Examples with ExternalView org.apache.helix.model.ExternalView used on opensource projects

Example 11 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class HelixExternalViewBasedRouting method processInstanceConfigChange.

public void processInstanceConfigChange() {
    long startTime = System.currentTimeMillis();
    // Get stats for all relevant instance configs
    HelixDataAccessor helixDataAccessor = _helixManager.getHelixDataAccessor();
    PropertyKey.Builder propertyKeyBuilder = helixDataAccessor.keyBuilder();
    List<String> instancesUsed = new ArrayList<>(_tablesForInstance.keySet());
    List<String> instancePaths = new ArrayList<>(instancesUsed.size());
    for (String instanceName : instancesUsed) {
        PropertyKey propertyKey = propertyKeyBuilder.instanceConfig(instanceName);
        instancePaths.add(propertyKey.getPath());
    }
    if (instancePaths.isEmpty()) {
        return;
    }
    long statFetchStart = System.currentTimeMillis();
    Stat[] instanceConfigStats = helixDataAccessor.getBaseDataAccessor().getStats(instancePaths, AccessOption.PERSISTENT);
    long statFetchEnd = System.currentTimeMillis();
    // Make a list of instance configs that changed
    long icConfigCheckStart = System.currentTimeMillis();
    List<String> instancesThatChanged = new ArrayList<>();
    for (int i = 0; i < instanceConfigStats.length; i++) {
        Stat instanceConfigStat = instanceConfigStats[i];
        if (instanceConfigStat != null) {
            String instanceName = instancesUsed.get(i);
            int currentInstanceConfigVersion = instanceConfigStat.getVersion();
            int lastKnownInstanceConfigVersion = _lastKnownInstanceConfigs.get(instanceName).getRecord().getVersion();
            if (currentInstanceConfigVersion != lastKnownInstanceConfigVersion) {
                instancesThatChanged.add(instanceName);
            }
        }
    }
    // Make a list of all tables affected by the instance config changes
    Set<String> affectedTables = new HashSet<>();
    for (String instanceName : instancesThatChanged) {
        affectedTables.addAll(_tablesForInstance.get(instanceName));
    }
    long icConfigCheckEnd = System.currentTimeMillis();
    // Update the routing tables
    long icFetchTime = 0;
    long evFetchTime = 0;
    long rebuildCheckTime = 0;
    long buildTime = 0;
    int routingTablesRebuiltCount = 0;
    if (!affectedTables.isEmpty()) {
        long icFetchStart = System.currentTimeMillis();
        List<InstanceConfig> instanceConfigs = helixDataAccessor.getChildValues(propertyKeyBuilder.instanceConfigs());
        long icFetchEnd = System.currentTimeMillis();
        icFetchTime = icFetchEnd - icFetchStart;
        for (String tableName : affectedTables) {
            long evFetchStart = System.currentTimeMillis();
            ExternalView externalView = helixDataAccessor.getProperty(propertyKeyBuilder.externalView(tableName));
            long evFetchEnd = System.currentTimeMillis();
            evFetchTime += evFetchEnd - evFetchStart;
            long rebuildCheckStart = System.currentTimeMillis();
            final boolean routingTableRebuildRequired = isRoutingTableRebuildRequired(tableName, externalView, instanceConfigs);
            long rebuildCheckEnd = System.currentTimeMillis();
            rebuildCheckTime += rebuildCheckEnd - rebuildCheckStart;
            if (routingTableRebuildRequired) {
                long rebuildStart = System.currentTimeMillis();
                buildRoutingTable(tableName, externalView, instanceConfigs);
                long rebuildEnd = System.currentTimeMillis();
                buildTime += rebuildEnd - rebuildStart;
                routingTablesRebuiltCount++;
            }
        }
    }
    long endTime = System.currentTimeMillis();
    LOGGER.info("Processed instance config change in {} ms (stat {} ms, IC check {} ms, IC fetch {} ms, EV fetch {} ms, rebuild check {} ms, rebuild {} ms), {} / {} routing tables rebuilt", (endTime - startTime), (statFetchEnd - statFetchStart), (icConfigCheckEnd - icConfigCheckStart), icFetchTime, evFetchTime, rebuildCheckTime, buildTime, routingTablesRebuiltCount, _lastKnownExternalViewVersionMap.size());
}

Example 12 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class RoutingTableTest method testHelixExternalViewBasedRoutingTable.

@Test
public void testHelixExternalViewBasedRoutingTable() throws Exception {
    RoutingTableBuilder routingStrategy = new RandomRoutingTableBuilder(100);
    HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, NO_LLC_ROUTING, null, new BaseConfiguration());
    routingTable.setSmallClusterRoutingTableBuilder(routingStrategy);
    ExternalView externalView = new ExternalView("testResource0_OFFLINE");
    externalView.setState("segment0", "dataServer_instance_0", "ONLINE");
    externalView.setState("segment0", "dataServer_instance_1", "ONLINE");
    externalView.setState("segment1", "dataServer_instance_1", "ONLINE");
    externalView.setState("segment1", "dataServer_instance_2", "ONLINE");
    externalView.setState("segment2", "dataServer_instance_2", "ONLINE");
    externalView.setState("segment2", "dataServer_instance_0", "ONLINE");
    List<InstanceConfig> instanceConfigs = generateInstanceConfigs("dataServer_instance", 0, 2);
    routingTable.markDataResourceOnline("testResource0_OFFLINE", externalView, instanceConfigs);
    ExternalView externalView1 = new ExternalView("testResource1_OFFLINE");
    externalView1.setState("segment10", "dataServer_instance_0", "ONLINE");
    externalView1.setState("segment11", "dataServer_instance_1", "ONLINE");
    externalView1.setState("segment12", "dataServer_instance_2", "ONLINE");
    routingTable.markDataResourceOnline("testResource1_OFFLINE", externalView1, instanceConfigs);
    ExternalView externalView2 = new ExternalView("testResource2_OFFLINE");
    externalView2.setState("segment20", "dataServer_instance_0", "ONLINE");
    externalView2.setState("segment21", "dataServer_instance_0", "ONLINE");
    externalView2.setState("segment22", "dataServer_instance_0", "ONLINE");
    externalView2.setState("segment20", "dataServer_instance_1", "ONLINE");
    externalView2.setState("segment21", "dataServer_instance_1", "ONLINE");
    externalView2.setState("segment22", "dataServer_instance_1", "ONLINE");
    externalView2.setState("segment20", "dataServer_instance_2", "ONLINE");
    externalView2.setState("segment21", "dataServer_instance_2", "ONLINE");
    externalView2.setState("segment22", "dataServer_instance_2", "ONLINE");
    routingTable.markDataResourceOnline("testResource2_OFFLINE", externalView2, instanceConfigs);
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource0_OFFLINE", "[segment0, segment1, segment2]", 3);
    }
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource1_OFFLINE", "[segment10, segment11, segment12]", 3);
    }
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource2_OFFLINE", "[segment20, segment21, segment22]", 3);
    }
}

Example 13 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class KafkaLowLevelConsumerRoutingTableBuilderTest method testAllOnlineRoutingTable.

@Test
public void testAllOnlineRoutingTable() {
    final int ITERATIONS = 1000;
    Random random = new Random();
    KafkaLowLevelConsumerRoutingTableBuilder routingTableBuilder = new KafkaLowLevelConsumerRoutingTableBuilder();
    routingTableBuilder.init(new BaseConfiguration());
    long totalNanos = 0L;
    for (int i = 0; i < ITERATIONS; i++) {
        // 3 to 14 instances
        int instanceCount = random.nextInt(12) + 3;
        // 4 to 11 partitions
        int partitionCount = random.nextInt(8) + 4;
        // 3 to 5 replicas
        int replicationFactor = random.nextInt(3) + 3;
        // Generate instances
        String[] instanceNames = new String[instanceCount];
        for (int serverInstanceId = 0; serverInstanceId < instanceCount; serverInstanceId++) {
            instanceNames[serverInstanceId] = "Server_localhost_" + serverInstanceId;
        }
        // Generate partitions
        String[][] segmentNames = new String[partitionCount][];
        int totalSegmentCount = 0;
        for (int partitionId = 0; partitionId < partitionCount; partitionId++) {
            // 0 to 31 segments in partition
            int segmentCount = random.nextInt(32);
            segmentNames[partitionId] = new String[segmentCount];
            for (int sequenceNumber = 0; sequenceNumber < segmentCount; sequenceNumber++) {
                segmentNames[partitionId][sequenceNumber] = new LLCSegmentName("table", partitionId, sequenceNumber, System.currentTimeMillis()).getSegmentName();
            }
            totalSegmentCount += segmentCount;
        }
        // Generate instance configurations
        List<InstanceConfig> instanceConfigs = new ArrayList<InstanceConfig>();
        for (String instanceName : instanceNames) {
            InstanceConfig instanceConfig = new InstanceConfig(instanceName);
            instanceConfigs.add(instanceConfig);
            instanceConfig.getRecord().setSimpleField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, "false");
        }
        // Generate a random external view
        ExternalView externalView = new ExternalView("table_REALTIME");
        int[] segmentCountForInstance = new int[instanceCount];
        int maxSegmentCountOnInstance = 0;
        for (int partitionId = 0; partitionId < segmentNames.length; partitionId++) {
            String[] segments = segmentNames[partitionId];
            // Assign each segment for this partition
            for (int replicaId = 0; replicaId < replicationFactor; ++replicaId) {
                for (int segmentIndex = 0; segmentIndex < segments.length; segmentIndex++) {
                    int instanceIndex = -1;
                    int randomOffset = random.nextInt(instanceCount);
                    // Pick the first random instance that has fewer than maxSegmentCountOnInstance segments assigned to it
                    for (int j = 0; j < instanceCount; j++) {
                        int potentialInstanceIndex = (j + randomOffset) % instanceCount;
                        if (segmentCountForInstance[potentialInstanceIndex] < maxSegmentCountOnInstance) {
                            instanceIndex = potentialInstanceIndex;
                            break;
                        }
                    }
                    // All replicas have exactly maxSegmentCountOnInstance, pick a replica and increment the max
                    if (instanceIndex == -1) {
                        maxSegmentCountOnInstance++;
                        instanceIndex = randomOffset;
                    }
                    // Increment the segment count for the instance
                    segmentCountForInstance[instanceIndex]++;
                    // Add the segment to the external view
                    externalView.setState(segmentNames[partitionId][segmentIndex], instanceNames[instanceIndex], "ONLINE");
                }
            }
        }
        // Create routing tables
        long startTime = System.nanoTime();
        List<ServerToSegmentSetMap> routingTables = routingTableBuilder.computeRoutingTableFromExternalView("table_REALTIME", externalView, instanceConfigs);
        long endTime = System.nanoTime();
        totalNanos += endTime - startTime;
        // Check that all routing tables generated match all segments, with no duplicates
        for (ServerToSegmentSetMap routingTable : routingTables) {
            Set<String> assignedSegments = new HashSet<String>();
            for (String server : routingTable.getServerSet()) {
                for (String segment : routingTable.getSegmentSet(server)) {
                    assertFalse(assignedSegments.contains(segment));
                    assignedSegments.add(segment);
                }
            }
            assertEquals(assignedSegments.size(), totalSegmentCount);
        }
    }
    LOGGER.warn("Routing table building avg ms: " + totalNanos / (ITERATIONS * 1000000.0));
}

Example 14 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class LargeClusterRoutingTableBuilderTest method validateAssertionForOneRoutingTable.

private void validateAssertionForOneRoutingTable(RoutingTableValidator routingTableValidator, String message, int instanceCount, int replicationFactor, int segmentCount) {
    final String tableName = "fakeTable_OFFLINE";
    ExternalView externalView = createExternalView(tableName, segmentCount, replicationFactor, instanceCount);
    List<InstanceConfig> instanceConfigs = createInstanceConfigs(instanceCount);
    validateAssertionForOneRoutingTable(routingTableValidator, message, externalView, instanceConfigs, tableName);
}

Example 15 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class DeleteOverlappingSegmentsInPinot method deleteOverlappingSegments.

public static boolean deleteOverlappingSegments(String zkUrl, String zkCluster, String tableName) {
    boolean updateSuccessful = false;
    if (!tableName.endsWith("_OFFLINE")) {
        tableName = tableName + "_OFFLINE";
    }
    ZkClient zkClient = new ZkClient(zkUrl);
    ZNRecordSerializer zkSerializer = new ZNRecordSerializer();
    zkClient.setZkSerializer(zkSerializer);
    BaseDataAccessor<ZNRecord> baseDataAccessor = new ZkBaseDataAccessor<>(zkClient);
    HelixDataAccessor helixDataAccessor = new ZKHelixDataAccessor(zkCluster, baseDataAccessor);
    Builder keyBuilder = helixDataAccessor.keyBuilder();
    PropertyKey idealStateKey = keyBuilder.idealStates(tableName);
    PropertyKey externalViewKey = keyBuilder.externalView(tableName);
    IdealState currentIdealState = helixDataAccessor.getProperty(idealStateKey);
    byte[] serializeIS = zkSerializer.serialize(currentIdealState.getRecord());
    String name = tableName + ".idealstate." + System.currentTimeMillis();
    File outputFile = new File("/tmp", name);
    try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
        IOUtils.write(serializeIS, fileOutputStream);
    } catch (IOException e) {
        LOG.error("Exception in delete overlapping segments", e);
        return updateSuccessful;
    }
    LOG.info("Saved current idealstate to {}", outputFile);
    IdealState newIdealState;
    do {
        newIdealState = computeNewIdealStateAfterDeletingOverlappingSegments(helixDataAccessor, idealStateKey);
        LOG.info("Updating IdealState");
        updateSuccessful = helixDataAccessor.getBaseDataAccessor().set(idealStateKey.getPath(), newIdealState.getRecord(), newIdealState.getRecord().getVersion(), AccessOption.PERSISTENT);
        if (updateSuccessful) {
            int numSegmentsDeleted = currentIdealState.getPartitionSet().size() - newIdealState.getPartitionSet().size();
            LOG.info("Successfully updated IdealState: Removed segments: {}", (numSegmentsDeleted));
        }
    } while (!updateSuccessful);
    try {
        while (true) {
            Thread.sleep(10000);
            ExternalView externalView = helixDataAccessor.getProperty(externalViewKey);
            IdealState idealState = helixDataAccessor.getProperty(idealStateKey);
            Set<String> evPartitionSet = externalView.getPartitionSet();
            Set<String> isPartitionSet = idealState.getPartitionSet();
            if (evPartitionSet.equals(isPartitionSet)) {
                LOG.info("Table {} has reached stable state. i.e segments in external view match idealstates", tableName);
                break;
            }
        }
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    return updateSuccessful;
}