Examples with ServerToSegmentSetMap com.linkedin.pinot.routing.ServerToSegmentSetMap used on opensource projects

Example 6 with ServerToSegmentSetMap

use of com.linkedin.pinot.routing.ServerToSegmentSetMap in project pinot by linkedin.

the class KafkaLowLevelConsumerRoutingTableBuilderTest method testMultipleConsumingSegments.

@Test
public void testMultipleConsumingSegments() {
    final int SEGMENT_COUNT = 10;
    final int ONLINE_SEGMENT_COUNT = 8;
    final int CONSUMING_SEGMENT_COUNT = SEGMENT_COUNT - ONLINE_SEGMENT_COUNT;
    KafkaLowLevelConsumerRoutingTableBuilder routingTableBuilder = new KafkaLowLevelConsumerRoutingTableBuilder();
    routingTableBuilder.init(new BaseConfiguration());
    List<SegmentName> segmentNames = new ArrayList<SegmentName>();
    for (int i = 0; i < SEGMENT_COUNT; ++i) {
        segmentNames.add(new LLCSegmentName("table", 0, i, System.currentTimeMillis()));
    }
    List<InstanceConfig> instanceConfigs = new ArrayList<InstanceConfig>();
    InstanceConfig instanceConfig = new InstanceConfig("Server_localhost_1234");
    instanceConfigs.add(instanceConfig);
    instanceConfig.getRecord().setSimpleField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, "false");
    // Generate an external view for a single server with some consuming segments
    ExternalView externalView = new ExternalView("table_REALTIME");
    for (int i = 0; i < ONLINE_SEGMENT_COUNT; i++) {
        externalView.setState(segmentNames.get(i).getSegmentName(), "Server_localhost_1234", "ONLINE");
    }
    for (int i = ONLINE_SEGMENT_COUNT; i < SEGMENT_COUNT; ++i) {
        externalView.setState(segmentNames.get(i).getSegmentName(), "Server_localhost_1234", "CONSUMING");
    }
    List<ServerToSegmentSetMap> routingTables = routingTableBuilder.computeRoutingTableFromExternalView("table", externalView, instanceConfigs);
    for (ServerToSegmentSetMap routingTable : routingTables) {
        for (String server : routingTable.getServerSet()) {
            Set<String> segmentSet = routingTable.getSegmentSet(server);
            assertEquals(segmentSet.size(), ONLINE_SEGMENT_COUNT + 1, "");
            // Should only contain the first consuming segment, not the second
            assertTrue(segmentSet.contains(segmentNames.get(ONLINE_SEGMENT_COUNT).getSegmentName()), "Segment set does not contain the first segment in consuming state");
            for (int i = ONLINE_SEGMENT_COUNT + 1; i < SEGMENT_COUNT; i++) {
                assertFalse(segmentSet.contains(segmentNames.get(i).getSegmentName()), "Segment set contains a segment in consuming state that should not be there");
            }
        }
    }
}

Example 7 with ServerToSegmentSetMap

use of com.linkedin.pinot.routing.ServerToSegmentSetMap in project pinot by linkedin.

the class LargeClusterRoutingTableBuilderTest method validateAssertionForOneRoutingTable.

private void validateAssertionForOneRoutingTable(RoutingTableValidator routingTableValidator, String message, ExternalView externalView, List<InstanceConfig> instanceConfigs, String tableName) {
    List<ServerToSegmentSetMap> routingTables = _largeClusterRoutingTableBuilder.computeRoutingTableFromExternalView(tableName, externalView, instanceConfigs);
    for (ServerToSegmentSetMap routingTable : routingTables) {
        boolean isValid = routingTableValidator.isRoutingTableValid(routingTable, externalView, instanceConfigs);
        if (!isValid) {
            System.out.println("externalView = " + externalView);
            System.out.println("routingTable = " + routingTable);
            System.out.println("RANDOM_SEED = " + RANDOM_SEED);
        }
        assertTrue(isValid, message);
    }
}

Example 8 with ServerToSegmentSetMap

use of com.linkedin.pinot.routing.ServerToSegmentSetMap in project pinot by linkedin.

the class LargeClusterRoutingTableBuilderTest method testRoutingTableSizeGenerallyHasConfiguredServerCount.

@Test
public void testRoutingTableSizeGenerallyHasConfiguredServerCount() {
    final String tableName = "fakeTable_OFFLINE";
    final int segmentCount = 100;
    final int replicationFactor = 10;
    final int instanceCount = 50;
    final int desiredServerCount = 20;
    ExternalView externalView = createExternalView(tableName, segmentCount, replicationFactor, instanceCount);
    List<InstanceConfig> instanceConfigs = createInstanceConfigs(instanceCount);
    List<ServerToSegmentSetMap> routingTables = _largeClusterRoutingTableBuilder.computeRoutingTableFromExternalView(tableName, externalView, instanceConfigs);
    int routingTableCount = 0;
    int largerThanDesiredRoutingTableCount = 0;
    for (ServerToSegmentSetMap routingTable : routingTables) {
        routingTableCount++;
        if (desiredServerCount < routingTable.getServerSet().size()) {
            largerThanDesiredRoutingTableCount++;
        }
    }
    assertTrue(largerThanDesiredRoutingTableCount / 0.6 < routingTableCount, "More than 60% of routing tables exceed the desired routing table size, RANDOM_SEED = " + RANDOM_SEED);
}

Example 9 with ServerToSegmentSetMap

use of com.linkedin.pinot.routing.ServerToSegmentSetMap in project pinot by linkedin.

the class LargeClusterRoutingTableBuilderTest method testRoutingTableExcludesDisabledAndRebootingInstances.

@Test
public void testRoutingTableExcludesDisabledAndRebootingInstances() {
    final String tableName = "fakeTable_OFFLINE";
    final int segmentCount = 100;
    final int replicationFactor = 6;
    final int instanceCount = 50;
    ExternalView externalView = createExternalView(tableName, segmentCount, replicationFactor, instanceCount);
    List<InstanceConfig> instanceConfigs = createInstanceConfigs(instanceCount);
    final InstanceConfig disabledHelixInstance = instanceConfigs.get(0);
    final String disabledHelixInstanceName = disabledHelixInstance.getInstanceName();
    disabledHelixInstance.setInstanceEnabled(false);
    final InstanceConfig shuttingDownInstance = instanceConfigs.get(1);
    final String shuttingDownInstanceName = shuttingDownInstance.getInstanceName();
    shuttingDownInstance.getRecord().setSimpleField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, Boolean.toString(true));
    validateAssertionForOneRoutingTable(new RoutingTableValidator() {

        @Override
        public boolean isRoutingTableValid(ServerToSegmentSetMap routingTable, ExternalView externalView, List<InstanceConfig> instanceConfigs) {
            for (String server : routingTable.getServerSet()) {
                // These servers should not appear in the routing table
                if (server.equals(disabledHelixInstanceName) || server.equals(shuttingDownInstanceName)) {
                    return false;
                }
            }
            return true;
        }
    }, "Routing table should not contain disabled instances", externalView, instanceConfigs, tableName);
}

Example 10 with ServerToSegmentSetMap

use of com.linkedin.pinot.routing.ServerToSegmentSetMap in project pinot by linkedin.

the class LargeClusterRoutingTableBuilderTest method testRoutingTableServerLoadIsRelativelyEqual.

@Test
public void testRoutingTableServerLoadIsRelativelyEqual() {
    final String tableName = "fakeTable_OFFLINE";
    final int segmentCount = 300;
    final int replicationFactor = 10;
    final int instanceCount = 50;
    ExternalView externalView = createExternalView(tableName, segmentCount, replicationFactor, instanceCount);
    List<InstanceConfig> instanceConfigs = createInstanceConfigs(instanceCount);
    List<ServerToSegmentSetMap> routingTables = _largeClusterRoutingTableBuilder.computeRoutingTableFromExternalView(tableName, externalView, instanceConfigs);
    Map<String, Integer> segmentCountPerServer = new HashMap<>();
    // Count number of segments assigned per server
    for (ServerToSegmentSetMap routingTable : routingTables) {
        for (String server : routingTable.getServerSet()) {
            Integer segmentCountForServer = segmentCountPerServer.get(server);
            if (segmentCountForServer == null) {
                segmentCountForServer = 0;
            }
            segmentCountForServer += routingTable.getSegmentSet(server).size();
            segmentCountPerServer.put(server, segmentCountForServer);
        }
    }
    int minNumberOfSegmentsAssignedPerServer = Integer.MAX_VALUE;
    int maxNumberOfSegmentsAssignedPerServer = 0;
    for (Integer segmentCountForServer : segmentCountPerServer.values()) {
        if (segmentCountForServer < minNumberOfSegmentsAssignedPerServer) {
            minNumberOfSegmentsAssignedPerServer = segmentCountForServer;
        }
        if (maxNumberOfSegmentsAssignedPerServer < segmentCountForServer) {
            maxNumberOfSegmentsAssignedPerServer = segmentCountForServer;
        }
    }
    assertTrue(maxNumberOfSegmentsAssignedPerServer < minNumberOfSegmentsAssignedPerServer * 1.5, "At least one server has more than 150% of the load of the least loaded server, minNumberOfSegmentsAssignedPerServer = " + minNumberOfSegmentsAssignedPerServer + " maxNumberOfSegmentsAssignedPerServer = " + maxNumberOfSegmentsAssignedPerServer + " RANDOM_SEED = " + RANDOM_SEED);
}