Examples with IdealState org.apache.helix.model.IdealState used on opensource projects

Example 31 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class RetentionManagerTest method testRealtimeLLCCleanup.

@Test
public void testRealtimeLLCCleanup() throws Exception {
    final int initialNumSegments = 8;
    final long now = System.currentTimeMillis();
    Set<String> remainingSegments = setupRealtimeTable(initialNumSegments, now);
    // At least one segment should be deleted, otherwise we don't have a test
    Assert.assertTrue(initialNumSegments - remainingSegments.size() > 0);
    _retentionManager = new RetentionManager(_pinotHelixResourceManager, 5);
    _retentionManager.start();
    // Do not check external view when validating because the segments that are OFFLINE in Idealstate to begin with
    // never show up in Externalview
    validate(initialNumSegments, _realtimeTableName, remainingSegments.size(), false);
    // Ensure that the segments that should be present are indeed present.
    IdealState idealState = HelixHelper.getTableIdealState(_helixZkManager, _realtimeTableName);
    for (final String segmentId : remainingSegments) {
        Assert.assertTrue(idealState.getPartitionSet().contains(segmentId));
        Assert.assertNotNull(ZKMetadataProvider.getRealtimeSegmentZKMetadata(_propertyStore, _realtimeTableName, segmentId));
    }
    cleanupSegments(_realtimeTableName);
}

Example 32 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class ValidationManagerTest method testLLCValidation.

@Test
public void testLLCValidation() throws Exception {
    final String topicName = "topic";
    final int kafkaPartitionCount = 2;
    final String realtimeTableName = "table_REALTIME";
    final String tableName = TableNameBuilder.extractRawTableName(realtimeTableName);
    // Server 1
    final String S1 = "S1";
    // Server 2
    final String S2 = "S2";
    // Server 3
    final String S3 = "S3";
    final List<String> hosts = Arrays.asList(new String[] { S1, S2, S3 });
    final HelixAdmin helixAdmin = _pinotHelixResourceManager.getHelixAdmin();
    ZNRecord znRecord = new ZNRecord(topicName);
    for (int i = 0; i < kafkaPartitionCount; i++) {
        znRecord.setListField(Integer.toString(i), hosts);
    }
    makeMockPinotLLCRealtimeSegmentManager(znRecord);
    long msSinceEpoch = 1540;
    LLCSegmentName p0s0 = new LLCSegmentName(tableName, 0, 0, msSinceEpoch);
    LLCSegmentName p0s1 = new LLCSegmentName(tableName, 0, 1, msSinceEpoch);
    LLCSegmentName p1s0 = new LLCSegmentName(tableName, 1, 0, msSinceEpoch);
    LLCSegmentName p1s1 = new LLCSegmentName(tableName, 1, 1, msSinceEpoch);
    IdealState idealstate = PinotTableIdealStateBuilder.buildEmptyIdealStateFor(realtimeTableName, 3);
    idealstate.setPartitionState(p0s0.getSegmentName(), S1, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    idealstate.setPartitionState(p0s0.getSegmentName(), S2, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    idealstate.setPartitionState(p0s0.getSegmentName(), S3, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    //    idealstate.setPartitionState(p0s1.getSegmentName(), S1, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    //    idealstate.setPartitionState(p0s1.getSegmentName(), S2, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    //    idealstate.setPartitionState(p0s1.getSegmentName(), S3, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    idealstate.setPartitionState(p1s0.getSegmentName(), S1, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    idealstate.setPartitionState(p1s0.getSegmentName(), S2, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    idealstate.setPartitionState(p1s0.getSegmentName(), S3, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    idealstate.setPartitionState(p1s1.getSegmentName(), S1, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    idealstate.setPartitionState(p1s1.getSegmentName(), S2, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    idealstate.setPartitionState(p1s1.getSegmentName(), S3, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    helixAdmin.addResource(HELIX_CLUSTER_NAME, realtimeTableName, idealstate);
    FakeValidationMetrics validationMetrics = new FakeValidationMetrics();
    ValidationManager validationManager = new ValidationManager(validationMetrics, _pinotHelixResourceManager, new ControllerConf(), _segmentManager);
    Map<String, String> streamConfigs = new HashMap<String, String>(4);
    streamConfigs.put(StringUtil.join(".", CommonConstants.Helix.DataSource.STREAM_PREFIX, CommonConstants.Helix.DataSource.Realtime.Kafka.CONSUMER_TYPE), "highLevel,simple");
    Field autoCreateOnError = ValidationManager.class.getDeclaredField("_autoCreateOnError");
    autoCreateOnError.setAccessible(true);
    autoCreateOnError.setBoolean(validationManager, false);
    AbstractTableConfig tableConfig = mock(AbstractTableConfig.class);
    IndexingConfig indexingConfig = mock(IndexingConfig.class);
    when(tableConfig.getIndexingConfig()).thenReturn(indexingConfig);
    when(indexingConfig.getStreamConfigs()).thenReturn(streamConfigs);
    validationManager.validateLLCSegments(realtimeTableName, tableConfig);
    Assert.assertEquals(validationMetrics.partitionCount, 1);
    // Set partition 0 to have one instance in CONSUMING state, and others in OFFLINE.
    // we should not flag any partitions to correct.
    helixAdmin.dropResource(HELIX_CLUSTER_NAME, realtimeTableName);
    idealstate.setPartitionState(p0s1.getSegmentName(), S1, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    idealstate.setPartitionState(p0s1.getSegmentName(), S2, PinotHelixSegmentOnlineOfflineStateModelGenerator.OFFLINE_STATE);
    idealstate.setPartitionState(p0s1.getSegmentName(), S3, PinotHelixSegmentOnlineOfflineStateModelGenerator.OFFLINE_STATE);
    helixAdmin.addResource(HELIX_CLUSTER_NAME, realtimeTableName, idealstate);
    validationManager.validateLLCSegments(realtimeTableName, tableConfig);
    Assert.assertEquals(validationMetrics.partitionCount, 0);
    helixAdmin.dropResource(HELIX_CLUSTER_NAME, realtimeTableName);
}

Example 33 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class RetentionManagerTest method registerSegmentMetadata.

private void registerSegmentMetadata(SegmentMetadata segmentMetadata) {
    // put into propertyStore
    OfflineSegmentZKMetadata offlineSegmentZKMetadata = new OfflineSegmentZKMetadata();
    ZKMetadataUtils.updateSegmentMetadata(offlineSegmentZKMetadata, segmentMetadata);
    ZKMetadataProvider.setOfflineSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), offlineSegmentZKMetadata);
    // put into idealStates
    IdealState idealState = _helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, _offlineTableName);
    idealState.setPartitionState(segmentMetadata.getName(), "Server_localhost_0", "ONLINE");
    idealState.setPartitionState(segmentMetadata.getName(), "Server_localhost_1", "ONLINE");
    _helixAdmin.setResourceIdealState(HELIX_CLUSTER_NAME, _offlineTableName, idealState);
}

Example 34 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class SegmentCompletionIntegrationTests method testStopConsumingToOfflineAndAutofix.

// Test that if we send stoppedConsuming to the controller, the segment goes offline.
@Test
public void testStopConsumingToOfflineAndAutofix() throws Exception {
    final String realtimeTableName = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(_tableName);
    long endTime = now() + MAX_RUN_TIME_SECONDS * 1000L;
    while (now() < endTime) {
        ExternalView ev = HelixHelper.getExternalViewForResource(_helixAdmin, _clusterName, realtimeTableName);
        if (ev != null) {
            Map<String, String> stateMap = ev.getStateMap(_segmentName);
            if (stateMap != null && stateMap.containsKey(_serverInstance)) {
                if (stateMap.get(_serverInstance).equals(PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE)) {
                    break;
                }
            }
        }
        Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
    }
    Assert.assertTrue(now() < endTime, "Failed trying to reach consuming state");
    // Now report to the controller that we had to stop consumption
    ServerSegmentCompletionProtocolHandler protocolHandler = new ServerSegmentCompletionProtocolHandler(_serverInstance);
    SegmentCompletionProtocol.Request.Params params = new SegmentCompletionProtocol.Request.Params();
    params.withOffset(45688L).withSegmentName(_segmentName).withReason("RandomReason");
    SegmentCompletionProtocol.Response response = protocolHandler.segmentStoppedConsuming(params);
    Assert.assertTrue(response.getStatus() == SegmentCompletionProtocol.ControllerResponseStatus.PROCESSED);
    while (now() < endTime) {
        ExternalView ev = HelixHelper.getExternalViewForResource(_helixAdmin, _clusterName, realtimeTableName);
        Map<String, String> stateMap = ev.getStateMap(_segmentName);
        if (stateMap.containsKey(_serverInstance)) {
            if (stateMap.get(_serverInstance).equals(PinotHelixSegmentOnlineOfflineStateModelGenerator.OFFLINE_STATE)) {
                break;
            }
        }
        Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
    }
    Assert.assertTrue(now() < endTime, "Failed trying to reach offline state");
    // Now call the validation manager, and the segment should fix itself
    getControllerValidationManager().runValidation();
    // Now there should be a new segment in CONSUMING state in the IDEALSTATE.
    IdealState idealState = HelixHelper.getTableIdealState(_helixManager, realtimeTableName);
    Assert.assertEquals(idealState.getPartitionSet().size(), 2);
    for (String segmentId : idealState.getPartitionSet()) {
        if (!segmentId.equals(_segmentName)) {
            // This is a new segment. Verify that it is in CONSUMING state, and has a sequence number 1 more than prev one
            LLCSegmentName oldSegmentName = new LLCSegmentName(_segmentName);
            LLCSegmentName newSegmentName = new LLCSegmentName(segmentId);
            Assert.assertEquals(newSegmentName.getSequenceNumber(), oldSegmentName.getSequenceNumber() + 1);
            Map<String, String> instanceStateMap = idealState.getInstanceStateMap(segmentId);
            for (String state : instanceStateMap.values()) {
                Assert.assertTrue(state.equals(PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE));
            }
        }
    }
// We will assume that it eventually makes it to externalview
}

Example 35 with IdealState

use of org.apache.helix.model.IdealState in project pinot by linkedin.

the class StarTreeClusterIntegrationTest method waitForExternalViewUpdate.

/**
   * Wait for External View to be in sync with Ideal State.
   * @return
   */
private boolean waitForExternalViewUpdate() {
    final ZKHelixAdmin helixAdmin = new ZKHelixAdmin(ZkStarter.DEFAULT_ZK_STR);
    ClusterStateVerifier.Verifier customVerifier = new ClusterStateVerifier.Verifier() {

        @Override
        public boolean verify() {
            String clusterName = getHelixClusterName();
            List<String> resourcesInCluster = helixAdmin.getResourcesInCluster(clusterName);
            LOGGER.info("Waiting for external view to update for resources: {} startTime: {}", resourcesInCluster, new Timestamp(System.currentTimeMillis()));
            for (String resourceName : resourcesInCluster) {
                IdealState idealState = helixAdmin.getResourceIdealState(clusterName, resourceName);
                ExternalView externalView = helixAdmin.getResourceExternalView(clusterName, resourceName);
                LOGGER.info("HERE for {},\n IS:{} \n EV:{}", resourceName, idealState, externalView);
                if (idealState == null || externalView == null) {
                    return false;
                }
                Set<String> partitionSet = idealState.getPartitionSet();
                for (String partition : partitionSet) {
                    Map<String, String> instanceStateMapIS = idealState.getInstanceStateMap(partition);
                    Map<String, String> instanceStateMapEV = externalView.getStateMap(partition);
                    if (instanceStateMapIS == null || instanceStateMapEV == null) {
                        return false;
                    }
                    if (!instanceStateMapIS.equals(instanceStateMapEV)) {
                        return false;
                    }
                }
                LOGGER.info("External View updated successfully for {},\n IS:{} \n EV:{}", resourceName, idealState, externalView);
            }
            LOGGER.info("External View updated successfully for {}", resourcesInCluster);
            return true;
        }
    };
    return ClusterStateVerifier.verifyByPolling(customVerifier, TIMEOUT_IN_MILLISECONDS);
}