Examples with ExternalView org.apache.helix.model.ExternalView used on opensource projects

Example 6 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class KafkaLowLevelConsumerRoutingTableBuilderTest method testAllOnlineRoutingTable.

@Test
public void testAllOnlineRoutingTable() {
    final int ITERATIONS = 1000;
    Random random = new Random();
    KafkaLowLevelConsumerRoutingTableBuilder routingTableBuilder = new KafkaLowLevelConsumerRoutingTableBuilder();
    routingTableBuilder.init(new BaseConfiguration());
    long totalNanos = 0L;
    for (int i = 0; i < ITERATIONS; i++) {
        // 3 to 14 instances
        int instanceCount = random.nextInt(12) + 3;
        // 4 to 11 partitions
        int partitionCount = random.nextInt(8) + 4;
        // 3 to 5 replicas
        int replicationFactor = random.nextInt(3) + 3;
        // Generate instances
        String[] instanceNames = new String[instanceCount];
        for (int serverInstanceId = 0; serverInstanceId < instanceCount; serverInstanceId++) {
            instanceNames[serverInstanceId] = "Server_localhost_" + serverInstanceId;
        }
        // Generate partitions
        String[][] segmentNames = new String[partitionCount][];
        int totalSegmentCount = 0;
        for (int partitionId = 0; partitionId < partitionCount; partitionId++) {
            // 0 to 31 segments in partition
            int segmentCount = random.nextInt(32);
            segmentNames[partitionId] = new String[segmentCount];
            for (int sequenceNumber = 0; sequenceNumber < segmentCount; sequenceNumber++) {
                segmentNames[partitionId][sequenceNumber] = new LLCSegmentName("table", partitionId, sequenceNumber, System.currentTimeMillis()).getSegmentName();
            }
            totalSegmentCount += segmentCount;
        }
        // Generate instance configurations
        List<InstanceConfig> instanceConfigs = new ArrayList<InstanceConfig>();
        for (String instanceName : instanceNames) {
            InstanceConfig instanceConfig = new InstanceConfig(instanceName);
            instanceConfigs.add(instanceConfig);
            instanceConfig.getRecord().setSimpleField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, "false");
        }
        // Generate a random external view
        ExternalView externalView = new ExternalView("table_REALTIME");
        int[] segmentCountForInstance = new int[instanceCount];
        int maxSegmentCountOnInstance = 0;
        for (int partitionId = 0; partitionId < segmentNames.length; partitionId++) {
            String[] segments = segmentNames[partitionId];
            // Assign each segment for this partition
            for (int replicaId = 0; replicaId < replicationFactor; ++replicaId) {
                for (int segmentIndex = 0; segmentIndex < segments.length; segmentIndex++) {
                    int instanceIndex = -1;
                    int randomOffset = random.nextInt(instanceCount);
                    // Pick the first random instance that has fewer than maxSegmentCountOnInstance segments assigned to it
                    for (int j = 0; j < instanceCount; j++) {
                        int potentialInstanceIndex = (j + randomOffset) % instanceCount;
                        if (segmentCountForInstance[potentialInstanceIndex] < maxSegmentCountOnInstance) {
                            instanceIndex = potentialInstanceIndex;
                            break;
                        }
                    }
                    // All replicas have exactly maxSegmentCountOnInstance, pick a replica and increment the max
                    if (instanceIndex == -1) {
                        maxSegmentCountOnInstance++;
                        instanceIndex = randomOffset;
                    }
                    // Increment the segment count for the instance
                    segmentCountForInstance[instanceIndex]++;
                    // Add the segment to the external view
                    externalView.setState(segmentNames[partitionId][segmentIndex], instanceNames[instanceIndex], "ONLINE");
                }
            }
        }
        // Create routing tables
        long startTime = System.nanoTime();
        List<ServerToSegmentSetMap> routingTables = routingTableBuilder.computeRoutingTableFromExternalView("table_REALTIME", externalView, instanceConfigs);
        long endTime = System.nanoTime();
        totalNanos += endTime - startTime;
        // Check that all routing tables generated match all segments, with no duplicates
        for (ServerToSegmentSetMap routingTable : routingTables) {
            Set<String> assignedSegments = new HashSet<String>();
            for (String server : routingTable.getServerSet()) {
                for (String segment : routingTable.getSegmentSet(server)) {
                    assertFalse(assignedSegments.contains(segment));
                    assignedSegments.add(segment);
                }
            }
            assertEquals(assignedSegments.size(), totalSegmentCount);
        }
    }
    LOGGER.warn("Routing table building avg ms: " + totalNanos / (ITERATIONS * 1000000.0));
}

Example 7 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class LargeClusterRoutingTableBuilderTest method validateAssertionForOneRoutingTable.

private void validateAssertionForOneRoutingTable(RoutingTableValidator routingTableValidator, String message, int instanceCount, int replicationFactor, int segmentCount) {
    final String tableName = "fakeTable_OFFLINE";
    ExternalView externalView = createExternalView(tableName, segmentCount, replicationFactor, instanceCount);
    List<InstanceConfig> instanceConfigs = createInstanceConfigs(instanceCount);
    validateAssertionForOneRoutingTable(routingTableValidator, message, externalView, instanceConfigs, tableName);
}

Example 8 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class DeleteOverlappingSegmentsInPinot method deleteOverlappingSegments.

public static boolean deleteOverlappingSegments(String zkUrl, String zkCluster, String tableName) {
    boolean updateSuccessful = false;
    if (!tableName.endsWith("_OFFLINE")) {
        tableName = tableName + "_OFFLINE";
    }
    ZkClient zkClient = new ZkClient(zkUrl);
    ZNRecordSerializer zkSerializer = new ZNRecordSerializer();
    zkClient.setZkSerializer(zkSerializer);
    BaseDataAccessor<ZNRecord> baseDataAccessor = new ZkBaseDataAccessor<>(zkClient);
    HelixDataAccessor helixDataAccessor = new ZKHelixDataAccessor(zkCluster, baseDataAccessor);
    Builder keyBuilder = helixDataAccessor.keyBuilder();
    PropertyKey idealStateKey = keyBuilder.idealStates(tableName);
    PropertyKey externalViewKey = keyBuilder.externalView(tableName);
    IdealState currentIdealState = helixDataAccessor.getProperty(idealStateKey);
    byte[] serializeIS = zkSerializer.serialize(currentIdealState.getRecord());
    String name = tableName + ".idealstate." + System.currentTimeMillis();
    File outputFile = new File("/tmp", name);
    try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
        IOUtils.write(serializeIS, fileOutputStream);
    } catch (IOException e) {
        LOG.error("Exception in delete overlapping segments", e);
        return updateSuccessful;
    }
    LOG.info("Saved current idealstate to {}", outputFile);
    IdealState newIdealState;
    do {
        newIdealState = computeNewIdealStateAfterDeletingOverlappingSegments(helixDataAccessor, idealStateKey);
        LOG.info("Updating IdealState");
        updateSuccessful = helixDataAccessor.getBaseDataAccessor().set(idealStateKey.getPath(), newIdealState.getRecord(), newIdealState.getRecord().getVersion(), AccessOption.PERSISTENT);
        if (updateSuccessful) {
            int numSegmentsDeleted = currentIdealState.getPartitionSet().size() - newIdealState.getPartitionSet().size();
            LOG.info("Successfully updated IdealState: Removed segments: {}", (numSegmentsDeleted));
        }
    } while (!updateSuccessful);
    try {
        while (true) {
            Thread.sleep(10000);
            ExternalView externalView = helixDataAccessor.getProperty(externalViewKey);
            IdealState idealState = helixDataAccessor.getProperty(idealStateKey);
            Set<String> evPartitionSet = externalView.getPartitionSet();
            Set<String> isPartitionSet = idealState.getPartitionSet();
            if (evPartitionSet.equals(isPartitionSet)) {
                LOG.info("Table {} has reached stable state. i.e segments in external view match idealstates", tableName);
                break;
            }
        }
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    return updateSuccessful;
}

Example 9 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class HelixBrokerStarterTest method testResourceAndTagAssignment.

@Test
public void testResourceAndTagAssignment() throws Exception {
    IdealState idealState;
    Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_BROKER").size(), 6);
    idealState = _helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
    Assert.assertEquals(idealState.getInstanceSet(DINING_TABLE_NAME).size(), SEGMENT_COUNT);
    ExternalView externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
    Assert.assertEquals(externalView.getStateMap(DINING_TABLE_NAME).size(), SEGMENT_COUNT);
    HelixExternalViewBasedRouting helixExternalViewBasedRouting = _helixBrokerStarter.getHelixExternalViewBasedRouting();
    Field brokerRoutingTableField;
    brokerRoutingTableField = HelixExternalViewBasedRouting.class.getDeclaredField("_brokerRoutingTable");
    brokerRoutingTableField.setAccessible(true);
    final Map<String, List<ServerToSegmentSetMap>> brokerRoutingTable = (Map<String, List<ServerToSegmentSetMap>>) brokerRoutingTableField.get(helixExternalViewBasedRouting);
    // Wait up to 30s for routing table to reach the expected size
    waitForPredicate(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return brokerRoutingTable.size() == 1;
        }
    }, 30000L);
    Assert.assertEquals(Arrays.toString(brokerRoutingTable.keySet().toArray()), "[dining_OFFLINE]");
    final String tableName = "coffee";
    JSONObject buildCreateOfflineTableV2JSON = ControllerRequestBuilderUtil.buildCreateOfflineTableJSON(tableName, "testServer", "testBroker", 1);
    AbstractTableConfig config = AbstractTableConfig.init(buildCreateOfflineTableV2JSON.toString());
    _pinotResourceManager.addTable(config);
    Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_BROKER").size(), 6);
    idealState = _helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
    Assert.assertEquals(idealState.getInstanceSet(COFFEE_TABLE_NAME).size(), SEGMENT_COUNT);
    Assert.assertEquals(idealState.getInstanceSet(DINING_TABLE_NAME).size(), SEGMENT_COUNT);
    // Wait up to 30s for broker external view to reach the expected size
    waitForPredicate(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE).getStateMap(COFFEE_TABLE_NAME).size() == SEGMENT_COUNT;
        }
    }, 30000L);
    externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
    Assert.assertEquals(externalView.getStateMap(COFFEE_TABLE_NAME).size(), SEGMENT_COUNT);
    // Wait up to 30s for routing table to reach the expected size
    waitForPredicate(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return brokerRoutingTable.size() == 2;
        }
    }, 30000L);
    Object[] tableArray = brokerRoutingTable.keySet().toArray();
    Arrays.sort(tableArray);
    Assert.assertEquals(Arrays.toString(tableArray), "[coffee_OFFLINE, dining_OFFLINE]");
    Set<String> serverSet = brokerRoutingTable.get(DINING_TABLE_NAME).get(0).getServerSet();
    Assert.assertEquals(brokerRoutingTable.get(DINING_TABLE_NAME).get(0).getSegmentSet(serverSet.iterator().next()).size(), 5);
    final String dataResource = DINING_TABLE_NAME;
    addOneSegment(dataResource);
    // Wait up to 30s for external view to reach the expected size
    waitForPredicate(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, DINING_TABLE_NAME).getPartitionSet().size() == SEGMENT_COUNT;
        }
    }, 30000L);
    externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, DINING_TABLE_NAME);
    Assert.assertEquals(externalView.getPartitionSet().size(), SEGMENT_COUNT);
    tableArray = brokerRoutingTable.keySet().toArray();
    Arrays.sort(tableArray);
    Assert.assertEquals(Arrays.toString(tableArray), "[coffee_OFFLINE, dining_OFFLINE]");
    // Wait up to 30s for routing table to reach the expected size
    waitForPredicate(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            ServerToSegmentSetMap routingTable = brokerRoutingTable.get(DINING_TABLE_NAME).get(0);
            String firstServer = routingTable.getServerSet().iterator().next();
            return routingTable.getSegmentSet(firstServer).size() == SEGMENT_COUNT;
        }
    }, 30000L);
    serverSet = brokerRoutingTable.get(DINING_TABLE_NAME).get(0).getServerSet();
    Assert.assertEquals(brokerRoutingTable.get(DINING_TABLE_NAME).get(0).getSegmentSet(serverSet.iterator().next()).size(), SEGMENT_COUNT);
}

Example 10 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class SegmentStatusChecker method runSegmentMetrics.

/**
   * Runs a segment status pass over the currently loaded tables.
   */
public void runSegmentMetrics() {
    if (!_pinotHelixResourceManager.isLeader()) {
        LOGGER.info("Skipping Segment Status check, not leader!");
        setStatusToDefault();
        stop();
        return;
    }
    long startTime = System.nanoTime();
    LOGGER.info("Starting Segment Status check for metrics");
    // Fetch the list of tables
    List<String> allTableNames = _pinotHelixResourceManager.getAllPinotTableNames();
    String helixClusterName = _pinotHelixResourceManager.getHelixClusterName();
    HelixAdmin helixAdmin = _pinotHelixResourceManager.getHelixAdmin();
    int realTimeTableCount = 0;
    int offlineTableCount = 0;
    ZkHelixPropertyStore<ZNRecord> propertyStore = _pinotHelixResourceManager.getPropertyStore();
    for (String tableName : allTableNames) {
        if (TableNameBuilder.getTableTypeFromTableName(tableName).equals(CommonConstants.Helix.TableType.OFFLINE)) {
            offlineTableCount++;
        } else {
            realTimeTableCount++;
        }
        IdealState idealState = helixAdmin.getResourceIdealState(helixClusterName, tableName);
        if ((idealState == null) || (idealState.getPartitionSet().isEmpty())) {
            _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS, 1);
            _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.PERCENT_OF_REPLICAS, 100);
            _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE, 100);
            continue;
        }
        _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.IDEALSTATE_ZNODE_SIZE, idealState.toString().length());
        ExternalView externalView = helixAdmin.getResourceExternalView(helixClusterName, tableName);
        // Keeps track of maximum number of replicas in ideal state
        int nReplicasIdealMax = 0;
        // Keeps track of minimum number of replicas in external view
        int nReplicasExternal = -1;
        // Keeps track of number of segments in error state
        int nErrors = 0;
        // Keeeps track of number segments with no online replicas
        int nOffline = 0;
        // Counts number of segments
        int nSegments = 0;
        for (String partitionName : idealState.getPartitionSet()) {
            int nReplicas = 0;
            int nIdeal = 0;
            nSegments++;
            // Skip segments not online in ideal state
            for (Map.Entry<String, String> serverAndState : idealState.getInstanceStateMap(partitionName).entrySet()) {
                if (serverAndState == null) {
                    break;
                }
                if (serverAndState.getValue().equals(ONLINE)) {
                    nIdeal++;
                    break;
                }
            }
            if (nIdeal == 0) {
                // No online segments in ideal state
                continue;
            }
            nReplicasIdealMax = (idealState.getInstanceStateMap(partitionName).size() > nReplicasIdealMax) ? idealState.getInstanceStateMap(partitionName).size() : nReplicasIdealMax;
            if ((externalView == null) || (externalView.getStateMap(partitionName) == null)) {
                // No replicas for this segment
                TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName);
                if ((tableType != null) && (tableType.equals(TableType.OFFLINE))) {
                    OfflineSegmentZKMetadata segmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(propertyStore, tableName, partitionName);
                    if (segmentZKMetadata != null && segmentZKMetadata.getPushTime() > System.currentTimeMillis() - _waitForPushTimeSeconds * 1000) {
                        // push not yet finished, skip
                        continue;
                    }
                }
                nOffline++;
                if (nOffline < MaxOfflineSegmentsToLog) {
                    LOGGER.warn("Segment {} of table {} has no replicas", partitionName, tableName);
                }
                nReplicasExternal = 0;
                continue;
            }
            for (Map.Entry<String, String> serverAndState : externalView.getStateMap(partitionName).entrySet()) {
                // Count number of online replicas
                if (serverAndState.getValue().equals(ONLINE)) {
                    nReplicas++;
                }
                if (serverAndState.getValue().equals(ERROR)) {
                    nErrors++;
                }
            }
            if (nReplicas == 0) {
                if (nOffline < MaxOfflineSegmentsToLog) {
                    LOGGER.warn("Segment {} of table {} has no online replicas", partitionName, tableName);
                }
                nOffline++;
            }
            nReplicasExternal = ((nReplicasExternal > nReplicas) || (nReplicasExternal == -1)) ? nReplicas : nReplicasExternal;
        }
        if (nReplicasExternal == -1) {
            nReplicasExternal = (nReplicasIdealMax == 0) ? 1 : 0;
        }
        // Synchronization provided by Controller Gauge to make sure that only one thread updates the gauge
        _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS, nReplicasExternal);
        _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.PERCENT_OF_REPLICAS, (nReplicasIdealMax > 0) ? (nReplicasExternal * 100 / nReplicasIdealMax) : 100);
        _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE, nErrors);
        _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE, (nSegments > 0) ? (100 - (nOffline * 100 / nSegments)) : 100);
        if (nOffline > 0) {
            LOGGER.warn("Table {} has {} segments with no online replicas", tableName, nOffline);
        }
        if (nReplicasExternal < nReplicasIdealMax) {
            LOGGER.warn("Table {} has {} replicas, below replication threshold :{}", tableName, nReplicasExternal, nReplicasIdealMax);
        }
    }
    _metricsRegistry.setValueOfGlobalGauge(ControllerGauge.REALTIME_TABLE_COUNT, realTimeTableCount);
    _metricsRegistry.setValueOfGlobalGauge(ControllerGauge.OFFLINE_TABLE_COUNT, offlineTableCount);
    long totalNanos = System.nanoTime() - startTime;
    LOGGER.info("Segment status metrics completed in {}ms", TimeUnit.MILLISECONDS.convert(totalNanos, TimeUnit.NANOSECONDS));
}