Examples with HelixAdmin org.apache.helix.HelixAdmin used on opensource projects

Example 6 with HelixAdmin

use of org.apache.helix.HelixAdmin in project pinot by linkedin.

the class SegmentStatusCheckerTest method noIdealState.

@Test
public void noIdealState() throws Exception {
    final String tableName = "myTable_REALTIME";
    List<String> allTableNames = new ArrayList<String>();
    allTableNames.add(tableName);
    IdealState idealState = null;
    HelixAdmin helixAdmin;
    {
        helixAdmin = mock(HelixAdmin.class);
        when(helixAdmin.getResourceIdealState("StatusChecker", tableName)).thenReturn(idealState);
        when(helixAdmin.getResourceExternalView("StatusChecker", tableName)).thenReturn(null);
    }
    {
        helixResourceManager = mock(PinotHelixResourceManager.class);
        when(helixResourceManager.isLeader()).thenReturn(true);
        when(helixResourceManager.getAllPinotTableNames()).thenReturn(allTableNames);
        when(helixResourceManager.getHelixClusterName()).thenReturn("StatusChecker");
        when(helixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
    }
    {
        config = mock(ControllerConf.class);
        when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
        when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
    }
    metricsRegistry = new MetricsRegistry();
    controllerMetrics = new ControllerMetrics(metricsRegistry);
    segmentStatusChecker = new SegmentStatusChecker(helixResourceManager, config);
    segmentStatusChecker.setMetricsRegistry(controllerMetrics);
    segmentStatusChecker.runSegmentMetrics();
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS), 1);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_OF_REPLICAS), 100);
    Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
    segmentStatusChecker.stop();
}

Example 7 with HelixAdmin

use of org.apache.helix.HelixAdmin in project pinot by linkedin.

the class SegmentDeletionManagerTest method testBulkDeleteWithFailures.

public void testBulkDeleteWithFailures(boolean useSet) throws Exception {
    HelixAdmin helixAdmin = makeHelixAdmin();
    ZkHelixPropertyStore<ZNRecord> propertyStore = makePropertyStore();
    FakeDeletionManager deletionManager = new FakeDeletionManager(helixAdmin, propertyStore);
    Collection<String> segments;
    if (useSet) {
        segments = new HashSet<String>();
    } else {
        segments = new ArrayList<String>();
    }
    segments.addAll(segmentsThatShouldBeDeleted());
    segments.addAll(segmentsInIdealStateOrExtView());
    segments.addAll(segmentsFailingPropStore());
    deletionManager.deleteSegmenetsFromPropertyStoreAndLocal(tableName, segments);
    Assert.assertTrue(deletionManager.segmentsToRetry.containsAll(segmentsFailingPropStore()));
    Assert.assertTrue(deletionManager.segmentsToRetry.containsAll(segmentsInIdealStateOrExtView()));
    Assert.assertTrue(deletionManager.segmentsRemovedFromStore.containsAll(segmentsThatShouldBeDeleted()));
}

Example 8 with HelixAdmin

use of org.apache.helix.HelixAdmin in project pinot by linkedin.

the class ValidationManagerTest method testLLCValidation.

@Test
public void testLLCValidation() throws Exception {
    final String topicName = "topic";
    final int kafkaPartitionCount = 2;
    final String realtimeTableName = "table_REALTIME";
    final String tableName = TableNameBuilder.extractRawTableName(realtimeTableName);
    // Server 1
    final String S1 = "S1";
    // Server 2
    final String S2 = "S2";
    // Server 3
    final String S3 = "S3";
    final List<String> hosts = Arrays.asList(new String[] { S1, S2, S3 });
    final HelixAdmin helixAdmin = _pinotHelixResourceManager.getHelixAdmin();
    ZNRecord znRecord = new ZNRecord(topicName);
    for (int i = 0; i < kafkaPartitionCount; i++) {
        znRecord.setListField(Integer.toString(i), hosts);
    }
    makeMockPinotLLCRealtimeSegmentManager(znRecord);
    long msSinceEpoch = 1540;
    LLCSegmentName p0s0 = new LLCSegmentName(tableName, 0, 0, msSinceEpoch);
    LLCSegmentName p0s1 = new LLCSegmentName(tableName, 0, 1, msSinceEpoch);
    LLCSegmentName p1s0 = new LLCSegmentName(tableName, 1, 0, msSinceEpoch);
    LLCSegmentName p1s1 = new LLCSegmentName(tableName, 1, 1, msSinceEpoch);
    IdealState idealstate = PinotTableIdealStateBuilder.buildEmptyIdealStateFor(realtimeTableName, 3);
    idealstate.setPartitionState(p0s0.getSegmentName(), S1, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    idealstate.setPartitionState(p0s0.getSegmentName(), S2, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    idealstate.setPartitionState(p0s0.getSegmentName(), S3, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    //    idealstate.setPartitionState(p0s1.getSegmentName(), S1, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    //    idealstate.setPartitionState(p0s1.getSegmentName(), S2, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    //    idealstate.setPartitionState(p0s1.getSegmentName(), S3, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    idealstate.setPartitionState(p1s0.getSegmentName(), S1, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    idealstate.setPartitionState(p1s0.getSegmentName(), S2, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    idealstate.setPartitionState(p1s0.getSegmentName(), S3, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    idealstate.setPartitionState(p1s1.getSegmentName(), S1, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    idealstate.setPartitionState(p1s1.getSegmentName(), S2, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    idealstate.setPartitionState(p1s1.getSegmentName(), S3, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    helixAdmin.addResource(HELIX_CLUSTER_NAME, realtimeTableName, idealstate);
    FakeValidationMetrics validationMetrics = new FakeValidationMetrics();
    ValidationManager validationManager = new ValidationManager(validationMetrics, _pinotHelixResourceManager, new ControllerConf(), _segmentManager);
    Map<String, String> streamConfigs = new HashMap<String, String>(4);
    streamConfigs.put(StringUtil.join(".", CommonConstants.Helix.DataSource.STREAM_PREFIX, CommonConstants.Helix.DataSource.Realtime.Kafka.CONSUMER_TYPE), "highLevel,simple");
    Field autoCreateOnError = ValidationManager.class.getDeclaredField("_autoCreateOnError");
    autoCreateOnError.setAccessible(true);
    autoCreateOnError.setBoolean(validationManager, false);
    AbstractTableConfig tableConfig = mock(AbstractTableConfig.class);
    IndexingConfig indexingConfig = mock(IndexingConfig.class);
    when(tableConfig.getIndexingConfig()).thenReturn(indexingConfig);
    when(indexingConfig.getStreamConfigs()).thenReturn(streamConfigs);
    validationManager.validateLLCSegments(realtimeTableName, tableConfig);
    Assert.assertEquals(validationMetrics.partitionCount, 1);
    // Set partition 0 to have one instance in CONSUMING state, and others in OFFLINE.
    // we should not flag any partitions to correct.
    helixAdmin.dropResource(HELIX_CLUSTER_NAME, realtimeTableName);
    idealstate.setPartitionState(p0s1.getSegmentName(), S1, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    idealstate.setPartitionState(p0s1.getSegmentName(), S2, PinotHelixSegmentOnlineOfflineStateModelGenerator.OFFLINE_STATE);
    idealstate.setPartitionState(p0s1.getSegmentName(), S3, PinotHelixSegmentOnlineOfflineStateModelGenerator.OFFLINE_STATE);
    helixAdmin.addResource(HELIX_CLUSTER_NAME, realtimeTableName, idealstate);
    validationManager.validateLLCSegments(realtimeTableName, tableConfig);
    Assert.assertEquals(validationMetrics.partitionCount, 0);
    helixAdmin.dropResource(HELIX_CLUSTER_NAME, realtimeTableName);
}

Example 9 with HelixAdmin

use of org.apache.helix.HelixAdmin in project ambry by linkedin.

the class HelixBootstrapUpgradeUtil method updatePartitionInfoIfChanged.

/**
 * Goes through each existing partition and updates the {@link PartitionState} and capacity information for the
 * replicas in each of the instances that hosts a replica for this partition, if it has changed and is different from
 * the current information in the static cluster map.
 * @param partition the partition whose {@link PartitionState} and/or capacity may have to be updated.
 */
private void updatePartitionInfoIfChanged(Partition partition) {
    for (Map.Entry<String, HelixAdmin> entry : adminForDc.entrySet()) {
        String dcName = entry.getKey();
        HelixAdmin dcAdmin = entry.getValue();
        String partitionName = Long.toString(partition.getId());
        long replicaCapacityInStatic = partition.getReplicaCapacityInBytes();
        boolean isSealed = partition.getPartitionState().equals(PartitionState.READ_ONLY);
        List<ReplicaId> replicaList = getReplicasInDc(partition, dcName);
        for (ReplicaId replicaId : replicaList) {
            DataNodeId node = replicaId.getDataNodeId();
            String instanceName = getInstanceName(node);
            InstanceConfig instanceConfig = dcAdmin.getInstanceConfig(clusterName, instanceName);
            boolean shouldSetInstanceConfig = false;
            if (updateSealedStateIfRequired(partitionName, instanceConfig, isSealed)) {
                System.out.println("Sealed state change of partition " + partitionName + " will be updated for instance " + instanceName);
                shouldSetInstanceConfig = true;
            }
            if (updateReplicaCapacityIfRequired(partitionName, instanceConfig, replicaId.getMountPath(), replicaCapacityInStatic)) {
                System.out.println("Replica capacity change of partition " + partitionName + " will be updated for instance" + instanceName);
                shouldSetInstanceConfig = true;
            }
            if (shouldSetInstanceConfig) {
                dcAdmin.setInstanceConfig(clusterName, instanceName, instanceConfig);
                System.out.println("Successfully updated InstanceConfig for instance " + instanceName);
            }
        }
    }
}

Example 10 with HelixAdmin

use of org.apache.helix.HelixAdmin in project ambry by linkedin.

the class HelixBootstrapUpgradeUtil method verifyEquivalencyWithStaticClusterMap.

/**
 * Verify that the information in Helix and the information in the static clustermap are equivalent.
 * @param hardwareLayout the {@link HardwareLayout} of the static clustermap.
 * @param partitionLayout the {@link PartitionLayout} of the static clustermap.
 */
private void verifyEquivalencyWithStaticClusterMap(HardwareLayout hardwareLayout, PartitionLayout partitionLayout) throws Exception {
    String clusterNameInStaticClusterMap = hardwareLayout.getClusterName();
    System.out.println("Verifying equivalency of static cluster: " + clusterNameInStaticClusterMap + " with the " + "corresponding cluster in Helix: " + clusterName);
    for (Datacenter dc : hardwareLayout.getDatacenters()) {
        HelixAdmin admin = adminForDc.get(dc.getName());
        ensureOrThrow(admin != null, "No ZkInfo for datacenter " + dc.getName());
        ensureOrThrow(admin.getClusters().contains(clusterName), "Cluster not found in ZK " + dataCenterToZkAddress.get(dc.getName()));
        verifyResourcesAndPartitionEquivalencyInDc(dc, clusterName, partitionLayout);
        verifyDataNodeAndDiskEquivalencyInDc(dc, clusterName, partitionLayout);
    }
    System.out.println("Successfully verified equivalency of static cluster: " + clusterNameInStaticClusterMap + " with the corresponding cluster in Helix: " + clusterName);
}