Examples with ServerToSegmentSetMap com.linkedin.pinot.routing.ServerToSegmentSetMap used on opensource projects

Example 1 with ServerToSegmentSetMap

use of com.linkedin.pinot.routing.ServerToSegmentSetMap in project pinot by linkedin.

the class BalancedRandomRoutingTableBuilder method computeRoutingTableFromExternalView.

@Override
public synchronized List<ServerToSegmentSetMap> computeRoutingTableFromExternalView(String tableName, ExternalView externalView, List<InstanceConfig> instanceConfigList) {
    RoutingTableInstancePruner pruner = new RoutingTableInstancePruner(instanceConfigList);
    List<Map<String, Set<String>>> routingTables = new ArrayList<Map<String, Set<String>>>();
    for (int i = 0; i < _numberOfRoutingTables; ++i) {
        routingTables.add(new HashMap<String, Set<String>>());
    }
    String[] segmentSet = externalView.getPartitionSet().toArray(new String[0]);
    for (String segment : segmentSet) {
        Map<String, String> instanceToStateMap = externalView.getStateMap(segment);
        for (String instance : instanceToStateMap.keySet().toArray(new String[0])) {
            if (!instanceToStateMap.get(instance).equals("ONLINE") || pruner.isInactive(instance)) {
                LOGGER.info("Removing offline/inactive instance '{}' from routing table computation", instance);
                instanceToStateMap.remove(instance);
            }
        }
        if (instanceToStateMap.size() > 0) {
            List<String> instanceList = new ArrayList<String>(instanceToStateMap.keySet());
            for (int i = 0; i < _numberOfRoutingTables; ++i) {
                Collections.shuffle(instanceList);
                String[] instances = instanceList.toArray(new String[instanceList.size()]);
                int minInstances = Integer.MAX_VALUE;
                int minIdx = -1;
                int base = 2;
                for (int k = 0; k < instances.length; ++k) {
                    int sizeOfCurrentInstance = 0;
                    if (routingTables.get(i).containsKey(instances[k])) {
                        sizeOfCurrentInstance = routingTables.get(i).get(instances[k]).size();
                    }
                    if (sizeOfCurrentInstance < minInstances) {
                        minInstances = sizeOfCurrentInstance;
                        minIdx = k;
                        base = 2;
                    }
                    if (sizeOfCurrentInstance == minInstances && (System.currentTimeMillis() % base == 0)) {
                        minIdx = k;
                        base = 2;
                    } else {
                        base++;
                    }
                }
                if (routingTables.get(i).containsKey(instances[minIdx])) {
                    routingTables.get(i).get(instances[minIdx]).add(segment);
                } else {
                    Set<String> instanceSegmentSet = new HashSet<String>();
                    instanceSegmentSet.add(segment);
                    routingTables.get(i).put(instances[minIdx], instanceSegmentSet);
                }
            }
        }
    }
    List<ServerToSegmentSetMap> resultRoutingTableList = new ArrayList<ServerToSegmentSetMap>();
    for (int i = 0; i < _numberOfRoutingTables; ++i) {
        resultRoutingTableList.add(new ServerToSegmentSetMap(routingTables.get(i)));
    }
    return resultRoutingTableList;
}

Example 2 with ServerToSegmentSetMap

use of com.linkedin.pinot.routing.ServerToSegmentSetMap in project pinot by linkedin.

the class KafkaLowLevelConsumerRoutingTableBuilderTest method testAllOnlineRoutingTable.

@Test
public void testAllOnlineRoutingTable() {
    final int ITERATIONS = 1000;
    Random random = new Random();
    KafkaLowLevelConsumerRoutingTableBuilder routingTableBuilder = new KafkaLowLevelConsumerRoutingTableBuilder();
    routingTableBuilder.init(new BaseConfiguration());
    long totalNanos = 0L;
    for (int i = 0; i < ITERATIONS; i++) {
        // 3 to 14 instances
        int instanceCount = random.nextInt(12) + 3;
        // 4 to 11 partitions
        int partitionCount = random.nextInt(8) + 4;
        // 3 to 5 replicas
        int replicationFactor = random.nextInt(3) + 3;
        // Generate instances
        String[] instanceNames = new String[instanceCount];
        for (int serverInstanceId = 0; serverInstanceId < instanceCount; serverInstanceId++) {
            instanceNames[serverInstanceId] = "Server_localhost_" + serverInstanceId;
        }
        // Generate partitions
        String[][] segmentNames = new String[partitionCount][];
        int totalSegmentCount = 0;
        for (int partitionId = 0; partitionId < partitionCount; partitionId++) {
            // 0 to 31 segments in partition
            int segmentCount = random.nextInt(32);
            segmentNames[partitionId] = new String[segmentCount];
            for (int sequenceNumber = 0; sequenceNumber < segmentCount; sequenceNumber++) {
                segmentNames[partitionId][sequenceNumber] = new LLCSegmentName("table", partitionId, sequenceNumber, System.currentTimeMillis()).getSegmentName();
            }
            totalSegmentCount += segmentCount;
        }
        // Generate instance configurations
        List<InstanceConfig> instanceConfigs = new ArrayList<InstanceConfig>();
        for (String instanceName : instanceNames) {
            InstanceConfig instanceConfig = new InstanceConfig(instanceName);
            instanceConfigs.add(instanceConfig);
            instanceConfig.getRecord().setSimpleField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, "false");
        }
        // Generate a random external view
        ExternalView externalView = new ExternalView("table_REALTIME");
        int[] segmentCountForInstance = new int[instanceCount];
        int maxSegmentCountOnInstance = 0;
        for (int partitionId = 0; partitionId < segmentNames.length; partitionId++) {
            String[] segments = segmentNames[partitionId];
            // Assign each segment for this partition
            for (int replicaId = 0; replicaId < replicationFactor; ++replicaId) {
                for (int segmentIndex = 0; segmentIndex < segments.length; segmentIndex++) {
                    int instanceIndex = -1;
                    int randomOffset = random.nextInt(instanceCount);
                    // Pick the first random instance that has fewer than maxSegmentCountOnInstance segments assigned to it
                    for (int j = 0; j < instanceCount; j++) {
                        int potentialInstanceIndex = (j + randomOffset) % instanceCount;
                        if (segmentCountForInstance[potentialInstanceIndex] < maxSegmentCountOnInstance) {
                            instanceIndex = potentialInstanceIndex;
                            break;
                        }
                    }
                    // All replicas have exactly maxSegmentCountOnInstance, pick a replica and increment the max
                    if (instanceIndex == -1) {
                        maxSegmentCountOnInstance++;
                        instanceIndex = randomOffset;
                    }
                    // Increment the segment count for the instance
                    segmentCountForInstance[instanceIndex]++;
                    // Add the segment to the external view
                    externalView.setState(segmentNames[partitionId][segmentIndex], instanceNames[instanceIndex], "ONLINE");
                }
            }
        }
        // Create routing tables
        long startTime = System.nanoTime();
        List<ServerToSegmentSetMap> routingTables = routingTableBuilder.computeRoutingTableFromExternalView("table_REALTIME", externalView, instanceConfigs);
        long endTime = System.nanoTime();
        totalNanos += endTime - startTime;
        // Check that all routing tables generated match all segments, with no duplicates
        for (ServerToSegmentSetMap routingTable : routingTables) {
            Set<String> assignedSegments = new HashSet<String>();
            for (String server : routingTable.getServerSet()) {
                for (String segment : routingTable.getSegmentSet(server)) {
                    assertFalse(assignedSegments.contains(segment));
                    assignedSegments.add(segment);
                }
            }
            assertEquals(assignedSegments.size(), totalSegmentCount);
        }
    }
    LOGGER.warn("Routing table building avg ms: " + totalNanos / (ITERATIONS * 1000000.0));
}

Example 3 with ServerToSegmentSetMap

use of com.linkedin.pinot.routing.ServerToSegmentSetMap in project pinot by linkedin.

the class KafkaHighLevelConsumerBasedRoutingTableBuilder method computeRoutingTableFromExternalView.

@Override
public List<ServerToSegmentSetMap> computeRoutingTableFromExternalView(String tableName, ExternalView externalView, List<InstanceConfig> instanceConfigList) {
    RoutingTableInstancePruner pruner = new RoutingTableInstancePruner(instanceConfigList);
    Set<String> segments = externalView.getPartitionSet();
    List<ServerToSegmentSetMap> routingTable = new ArrayList<ServerToSegmentSetMap>();
    Map<String, Map<String, Set<String>>> groupIdToRouting = new HashMap<String, Map<String, Set<String>>>();
    for (String segment : segments) {
        Map<String, String> instanceMap = externalView.getStateMap(segment);
        for (String instance : instanceMap.keySet()) {
            if (!instanceMap.get(instance).equals(CommonConstants.Helix.StateModel.SegmentOnlineOfflineStateModel.ONLINE) || pruner.isInactive(instance)) {
                continue;
            }
            // Skip segments that are not high level consumer segments
            if (!SegmentName.isHighLevelConsumerSegmentName(segment))
                continue;
            HLCSegmentName hlcSegmentName = new HLCSegmentName(segment);
            String groupId = hlcSegmentName.getGroupId();
            if (!groupIdToRouting.containsKey(groupId)) {
                groupIdToRouting.put(groupId, new HashMap<String, Set<String>>());
            }
            if (!groupIdToRouting.get(groupId).containsKey(instance)) {
                groupIdToRouting.get(groupId).put(instance, new HashSet<String>());
            }
            groupIdToRouting.get(groupId).get(instance).add(segment);
        }
    }
    for (Map<String, Set<String>> replicaRouting : groupIdToRouting.values()) {
        routingTable.add(new ServerToSegmentSetMap(replicaRouting));
    }
    return routingTable;
}

Example 4 with ServerToSegmentSetMap

use of com.linkedin.pinot.routing.ServerToSegmentSetMap in project pinot by linkedin.

the class HelixBrokerStarterTest method testResourceAndTagAssignment.

@Test
public void testResourceAndTagAssignment() throws Exception {
    IdealState idealState;
    Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_BROKER").size(), 6);
    idealState = _helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
    Assert.assertEquals(idealState.getInstanceSet(DINING_TABLE_NAME).size(), SEGMENT_COUNT);
    ExternalView externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
    Assert.assertEquals(externalView.getStateMap(DINING_TABLE_NAME).size(), SEGMENT_COUNT);
    HelixExternalViewBasedRouting helixExternalViewBasedRouting = _helixBrokerStarter.getHelixExternalViewBasedRouting();
    Field brokerRoutingTableField;
    brokerRoutingTableField = HelixExternalViewBasedRouting.class.getDeclaredField("_brokerRoutingTable");
    brokerRoutingTableField.setAccessible(true);
    final Map<String, List<ServerToSegmentSetMap>> brokerRoutingTable = (Map<String, List<ServerToSegmentSetMap>>) brokerRoutingTableField.get(helixExternalViewBasedRouting);
    // Wait up to 30s for routing table to reach the expected size
    waitForPredicate(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return brokerRoutingTable.size() == 1;
        }
    }, 30000L);
    Assert.assertEquals(Arrays.toString(brokerRoutingTable.keySet().toArray()), "[dining_OFFLINE]");
    final String tableName = "coffee";
    JSONObject buildCreateOfflineTableV2JSON = ControllerRequestBuilderUtil.buildCreateOfflineTableJSON(tableName, "testServer", "testBroker", 1);
    AbstractTableConfig config = AbstractTableConfig.init(buildCreateOfflineTableV2JSON.toString());
    _pinotResourceManager.addTable(config);
    Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_BROKER").size(), 6);
    idealState = _helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
    Assert.assertEquals(idealState.getInstanceSet(COFFEE_TABLE_NAME).size(), SEGMENT_COUNT);
    Assert.assertEquals(idealState.getInstanceSet(DINING_TABLE_NAME).size(), SEGMENT_COUNT);
    // Wait up to 30s for broker external view to reach the expected size
    waitForPredicate(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE).getStateMap(COFFEE_TABLE_NAME).size() == SEGMENT_COUNT;
        }
    }, 30000L);
    externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
    Assert.assertEquals(externalView.getStateMap(COFFEE_TABLE_NAME).size(), SEGMENT_COUNT);
    // Wait up to 30s for routing table to reach the expected size
    waitForPredicate(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return brokerRoutingTable.size() == 2;
        }
    }, 30000L);
    Object[] tableArray = brokerRoutingTable.keySet().toArray();
    Arrays.sort(tableArray);
    Assert.assertEquals(Arrays.toString(tableArray), "[coffee_OFFLINE, dining_OFFLINE]");
    Set<String> serverSet = brokerRoutingTable.get(DINING_TABLE_NAME).get(0).getServerSet();
    Assert.assertEquals(brokerRoutingTable.get(DINING_TABLE_NAME).get(0).getSegmentSet(serverSet.iterator().next()).size(), 5);
    final String dataResource = DINING_TABLE_NAME;
    addOneSegment(dataResource);
    // Wait up to 30s for external view to reach the expected size
    waitForPredicate(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, DINING_TABLE_NAME).getPartitionSet().size() == SEGMENT_COUNT;
        }
    }, 30000L);
    externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, DINING_TABLE_NAME);
    Assert.assertEquals(externalView.getPartitionSet().size(), SEGMENT_COUNT);
    tableArray = brokerRoutingTable.keySet().toArray();
    Arrays.sort(tableArray);
    Assert.assertEquals(Arrays.toString(tableArray), "[coffee_OFFLINE, dining_OFFLINE]");
    // Wait up to 30s for routing table to reach the expected size
    waitForPredicate(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            ServerToSegmentSetMap routingTable = brokerRoutingTable.get(DINING_TABLE_NAME).get(0);
            String firstServer = routingTable.getServerSet().iterator().next();
            return routingTable.getSegmentSet(firstServer).size() == SEGMENT_COUNT;
        }
    }, 30000L);
    serverSet = brokerRoutingTable.get(DINING_TABLE_NAME).get(0).getServerSet();
    Assert.assertEquals(brokerRoutingTable.get(DINING_TABLE_NAME).get(0).getSegmentSet(serverSet.iterator().next()).size(), SEGMENT_COUNT);
}

Example 5 with ServerToSegmentSetMap

use of com.linkedin.pinot.routing.ServerToSegmentSetMap in project pinot by linkedin.

the class BalancedRandomRoutingTableBuilderTest method isRandom.

@Test
public void isRandom() {
    // Build dummy external view
    BalancedRandomRoutingTableBuilder routingTableBuilder = new BalancedRandomRoutingTableBuilder();
    List<InstanceConfig> instanceConfigList = new ArrayList<>();
    ExternalView externalView = new ExternalView("dummy");
    externalView.setState("segment_1", "Server_1.2.3.4_1234", "ONLINE");
    externalView.setState("segment_1", "Server_1.2.3.5_2345", "ONLINE");
    externalView.setState("segment_1", "Server_1.2.3.6_3456", "ONLINE");
    externalView.setState("segment_2", "Server_1.2.3.4_1234", "ONLINE");
    externalView.setState("segment_2", "Server_1.2.3.5_2345", "ONLINE");
    externalView.setState("segment_2", "Server_1.2.3.6_3456", "ONLINE");
    externalView.setState("segment_3", "Server_1.2.3.4_1234", "ONLINE");
    externalView.setState("segment_3", "Server_1.2.3.5_2345", "ONLINE");
    externalView.setState("segment_3", "Server_1.2.3.6_3456", "ONLINE");
    // Create configs for above instances.
    instanceConfigList.add(new InstanceConfig("Server_1.2.3.4_1234"));
    instanceConfigList.add(new InstanceConfig("Server_1.2.3.5_2345"));
    instanceConfigList.add(new InstanceConfig("Server_1.2.3.6_3456"));
    // Build routing table
    List<ServerToSegmentSetMap> routingTable = routingTableBuilder.computeRoutingTableFromExternalView("dummy", externalView, instanceConfigList);
    // Check that at least two routing tables are different
    Iterator<ServerToSegmentSetMap> routingTableIterator = routingTable.iterator();
    ServerToSegmentSetMap previous = routingTableIterator.next();
    while (routingTableIterator.hasNext()) {
        ServerToSegmentSetMap current = routingTableIterator.next();
        //      System.out.println("previous = " + previous);
        if (!current.equals(previous)) {
            // Routing tables differ, test is successful
            return;
        }
    }
    Assert.fail("All routing tables are equal!");
}