Examples with InstanceConfig org.apache.helix.model.InstanceConfig used on opensource projects

Example 6 with InstanceConfig

use of org.apache.helix.model.InstanceConfig in project pinot by linkedin.

the class HelixExternalViewBasedRouting method processInstanceConfigChange.

public void processInstanceConfigChange() {
    long startTime = System.currentTimeMillis();
    // Get stats for all relevant instance configs
    HelixDataAccessor helixDataAccessor = _helixManager.getHelixDataAccessor();
    PropertyKey.Builder propertyKeyBuilder = helixDataAccessor.keyBuilder();
    List<String> instancesUsed = new ArrayList<>(_tablesForInstance.keySet());
    List<String> instancePaths = new ArrayList<>(instancesUsed.size());
    for (String instanceName : instancesUsed) {
        PropertyKey propertyKey = propertyKeyBuilder.instanceConfig(instanceName);
        instancePaths.add(propertyKey.getPath());
    }
    if (instancePaths.isEmpty()) {
        return;
    }
    long statFetchStart = System.currentTimeMillis();
    Stat[] instanceConfigStats = helixDataAccessor.getBaseDataAccessor().getStats(instancePaths, AccessOption.PERSISTENT);
    long statFetchEnd = System.currentTimeMillis();
    // Make a list of instance configs that changed
    long icConfigCheckStart = System.currentTimeMillis();
    List<String> instancesThatChanged = new ArrayList<>();
    for (int i = 0; i < instanceConfigStats.length; i++) {
        Stat instanceConfigStat = instanceConfigStats[i];
        if (instanceConfigStat != null) {
            String instanceName = instancesUsed.get(i);
            int currentInstanceConfigVersion = instanceConfigStat.getVersion();
            int lastKnownInstanceConfigVersion = _lastKnownInstanceConfigs.get(instanceName).getRecord().getVersion();
            if (currentInstanceConfigVersion != lastKnownInstanceConfigVersion) {
                instancesThatChanged.add(instanceName);
            }
        }
    }
    // Make a list of all tables affected by the instance config changes
    Set<String> affectedTables = new HashSet<>();
    for (String instanceName : instancesThatChanged) {
        affectedTables.addAll(_tablesForInstance.get(instanceName));
    }
    long icConfigCheckEnd = System.currentTimeMillis();
    // Update the routing tables
    long icFetchTime = 0;
    long evFetchTime = 0;
    long rebuildCheckTime = 0;
    long buildTime = 0;
    int routingTablesRebuiltCount = 0;
    if (!affectedTables.isEmpty()) {
        long icFetchStart = System.currentTimeMillis();
        List<InstanceConfig> instanceConfigs = helixDataAccessor.getChildValues(propertyKeyBuilder.instanceConfigs());
        long icFetchEnd = System.currentTimeMillis();
        icFetchTime = icFetchEnd - icFetchStart;
        for (String tableName : affectedTables) {
            long evFetchStart = System.currentTimeMillis();
            ExternalView externalView = helixDataAccessor.getProperty(propertyKeyBuilder.externalView(tableName));
            long evFetchEnd = System.currentTimeMillis();
            evFetchTime += evFetchEnd - evFetchStart;
            long rebuildCheckStart = System.currentTimeMillis();
            final boolean routingTableRebuildRequired = isRoutingTableRebuildRequired(tableName, externalView, instanceConfigs);
            long rebuildCheckEnd = System.currentTimeMillis();
            rebuildCheckTime += rebuildCheckEnd - rebuildCheckStart;
            if (routingTableRebuildRequired) {
                long rebuildStart = System.currentTimeMillis();
                buildRoutingTable(tableName, externalView, instanceConfigs);
                long rebuildEnd = System.currentTimeMillis();
                buildTime += rebuildEnd - rebuildStart;
                routingTablesRebuiltCount++;
            }
        }
    }
    long endTime = System.currentTimeMillis();
    LOGGER.info("Processed instance config change in {} ms (stat {} ms, IC check {} ms, IC fetch {} ms, EV fetch {} ms, rebuild check {} ms, rebuild {} ms), {} / {} routing tables rebuilt", (endTime - startTime), (statFetchEnd - statFetchStart), (icConfigCheckEnd - icConfigCheckStart), icFetchTime, evFetchTime, rebuildCheckTime, buildTime, routingTablesRebuiltCount, _lastKnownExternalViewVersionMap.size());
}

Example 7 with InstanceConfig

use of org.apache.helix.model.InstanceConfig in project pinot by linkedin.

the class RoutingTableTest method generateInstanceConfigs.

/**
   * Helper method to generate instance config lists. Instance names are generated as prefix_i, where
   * i ranges from start to end.
   *
   * @param prefix Instance name prefix
   * @param start Start index
   * @param end End index
   * @return
   */
private List<InstanceConfig> generateInstanceConfigs(String prefix, int start, int end) {
    List<InstanceConfig> configs = new ArrayList<>();
    for (int i = start; i <= end; ++i) {
        String instance = prefix + "_" + i;
        configs.add(new InstanceConfig(instance));
    }
    return configs;
}

Example 8 with InstanceConfig

use of org.apache.helix.model.InstanceConfig in project pinot by linkedin.

the class RoutingTableTest method testHelixExternalViewBasedRoutingTable.

@Test
public void testHelixExternalViewBasedRoutingTable() throws Exception {
    RoutingTableBuilder routingStrategy = new RandomRoutingTableBuilder(100);
    HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, NO_LLC_ROUTING, null, new BaseConfiguration());
    routingTable.setSmallClusterRoutingTableBuilder(routingStrategy);
    ExternalView externalView = new ExternalView("testResource0_OFFLINE");
    externalView.setState("segment0", "dataServer_instance_0", "ONLINE");
    externalView.setState("segment0", "dataServer_instance_1", "ONLINE");
    externalView.setState("segment1", "dataServer_instance_1", "ONLINE");
    externalView.setState("segment1", "dataServer_instance_2", "ONLINE");
    externalView.setState("segment2", "dataServer_instance_2", "ONLINE");
    externalView.setState("segment2", "dataServer_instance_0", "ONLINE");
    List<InstanceConfig> instanceConfigs = generateInstanceConfigs("dataServer_instance", 0, 2);
    routingTable.markDataResourceOnline("testResource0_OFFLINE", externalView, instanceConfigs);
    ExternalView externalView1 = new ExternalView("testResource1_OFFLINE");
    externalView1.setState("segment10", "dataServer_instance_0", "ONLINE");
    externalView1.setState("segment11", "dataServer_instance_1", "ONLINE");
    externalView1.setState("segment12", "dataServer_instance_2", "ONLINE");
    routingTable.markDataResourceOnline("testResource1_OFFLINE", externalView1, instanceConfigs);
    ExternalView externalView2 = new ExternalView("testResource2_OFFLINE");
    externalView2.setState("segment20", "dataServer_instance_0", "ONLINE");
    externalView2.setState("segment21", "dataServer_instance_0", "ONLINE");
    externalView2.setState("segment22", "dataServer_instance_0", "ONLINE");
    externalView2.setState("segment20", "dataServer_instance_1", "ONLINE");
    externalView2.setState("segment21", "dataServer_instance_1", "ONLINE");
    externalView2.setState("segment22", "dataServer_instance_1", "ONLINE");
    externalView2.setState("segment20", "dataServer_instance_2", "ONLINE");
    externalView2.setState("segment21", "dataServer_instance_2", "ONLINE");
    externalView2.setState("segment22", "dataServer_instance_2", "ONLINE");
    routingTable.markDataResourceOnline("testResource2_OFFLINE", externalView2, instanceConfigs);
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource0_OFFLINE", "[segment0, segment1, segment2]", 3);
    }
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource1_OFFLINE", "[segment10, segment11, segment12]", 3);
    }
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource2_OFFLINE", "[segment20, segment21, segment22]", 3);
    }
}

Example 9 with InstanceConfig

use of org.apache.helix.model.InstanceConfig in project pinot by linkedin.

the class KafkaLowLevelConsumerRoutingTableBuilderTest method testAllOnlineRoutingTable.

@Test
public void testAllOnlineRoutingTable() {
    final int ITERATIONS = 1000;
    Random random = new Random();
    KafkaLowLevelConsumerRoutingTableBuilder routingTableBuilder = new KafkaLowLevelConsumerRoutingTableBuilder();
    routingTableBuilder.init(new BaseConfiguration());
    long totalNanos = 0L;
    for (int i = 0; i < ITERATIONS; i++) {
        // 3 to 14 instances
        int instanceCount = random.nextInt(12) + 3;
        // 4 to 11 partitions
        int partitionCount = random.nextInt(8) + 4;
        // 3 to 5 replicas
        int replicationFactor = random.nextInt(3) + 3;
        // Generate instances
        String[] instanceNames = new String[instanceCount];
        for (int serverInstanceId = 0; serverInstanceId < instanceCount; serverInstanceId++) {
            instanceNames[serverInstanceId] = "Server_localhost_" + serverInstanceId;
        }
        // Generate partitions
        String[][] segmentNames = new String[partitionCount][];
        int totalSegmentCount = 0;
        for (int partitionId = 0; partitionId < partitionCount; partitionId++) {
            // 0 to 31 segments in partition
            int segmentCount = random.nextInt(32);
            segmentNames[partitionId] = new String[segmentCount];
            for (int sequenceNumber = 0; sequenceNumber < segmentCount; sequenceNumber++) {
                segmentNames[partitionId][sequenceNumber] = new LLCSegmentName("table", partitionId, sequenceNumber, System.currentTimeMillis()).getSegmentName();
            }
            totalSegmentCount += segmentCount;
        }
        // Generate instance configurations
        List<InstanceConfig> instanceConfigs = new ArrayList<InstanceConfig>();
        for (String instanceName : instanceNames) {
            InstanceConfig instanceConfig = new InstanceConfig(instanceName);
            instanceConfigs.add(instanceConfig);
            instanceConfig.getRecord().setSimpleField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, "false");
        }
        // Generate a random external view
        ExternalView externalView = new ExternalView("table_REALTIME");
        int[] segmentCountForInstance = new int[instanceCount];
        int maxSegmentCountOnInstance = 0;
        for (int partitionId = 0; partitionId < segmentNames.length; partitionId++) {
            String[] segments = segmentNames[partitionId];
            // Assign each segment for this partition
            for (int replicaId = 0; replicaId < replicationFactor; ++replicaId) {
                for (int segmentIndex = 0; segmentIndex < segments.length; segmentIndex++) {
                    int instanceIndex = -1;
                    int randomOffset = random.nextInt(instanceCount);
                    // Pick the first random instance that has fewer than maxSegmentCountOnInstance segments assigned to it
                    for (int j = 0; j < instanceCount; j++) {
                        int potentialInstanceIndex = (j + randomOffset) % instanceCount;
                        if (segmentCountForInstance[potentialInstanceIndex] < maxSegmentCountOnInstance) {
                            instanceIndex = potentialInstanceIndex;
                            break;
                        }
                    }
                    // All replicas have exactly maxSegmentCountOnInstance, pick a replica and increment the max
                    if (instanceIndex == -1) {
                        maxSegmentCountOnInstance++;
                        instanceIndex = randomOffset;
                    }
                    // Increment the segment count for the instance
                    segmentCountForInstance[instanceIndex]++;
                    // Add the segment to the external view
                    externalView.setState(segmentNames[partitionId][segmentIndex], instanceNames[instanceIndex], "ONLINE");
                }
            }
        }
        // Create routing tables
        long startTime = System.nanoTime();
        List<ServerToSegmentSetMap> routingTables = routingTableBuilder.computeRoutingTableFromExternalView("table_REALTIME", externalView, instanceConfigs);
        long endTime = System.nanoTime();
        totalNanos += endTime - startTime;
        // Check that all routing tables generated match all segments, with no duplicates
        for (ServerToSegmentSetMap routingTable : routingTables) {
            Set<String> assignedSegments = new HashSet<String>();
            for (String server : routingTable.getServerSet()) {
                for (String segment : routingTable.getSegmentSet(server)) {
                    assertFalse(assignedSegments.contains(segment));
                    assignedSegments.add(segment);
                }
            }
            assertEquals(assignedSegments.size(), totalSegmentCount);
        }
    }
    LOGGER.warn("Routing table building avg ms: " + totalNanos / (ITERATIONS * 1000000.0));
}

Example 10 with InstanceConfig

use of org.apache.helix.model.InstanceConfig in project pinot by linkedin.

the class LargeClusterRoutingTableBuilderTest method validateAssertionForOneRoutingTable.

private void validateAssertionForOneRoutingTable(RoutingTableValidator routingTableValidator, String message, int instanceCount, int replicationFactor, int segmentCount) {
    final String tableName = "fakeTable_OFFLINE";
    ExternalView externalView = createExternalView(tableName, segmentCount, replicationFactor, instanceCount);
    List<InstanceConfig> instanceConfigs = createInstanceConfigs(instanceCount);
    validateAssertionForOneRoutingTable(routingTableValidator, message, externalView, instanceConfigs, tableName);
}