Examples with CustomizedView org.apache.helix.model.CustomizedView used on opensource projects

Example 1 with CustomizedView

use of org.apache.helix.model.CustomizedView in project helix by apache.

the class TestRoutingTableProvider method testCustomizedViewCorrectConstructor.

@Test(dependsOnMethods = "testCustomizedViewWithoutType")
public void testCustomizedViewCorrectConstructor() throws Exception {
    Map<PropertyType, List<String>> sourceDataTypes = new HashMap<>();
    sourceDataTypes.put(PropertyType.CUSTOMIZEDVIEW, Arrays.asList("typeA"));
    MockRoutingTableProvider routingTableProvider = new MockRoutingTableProvider(_spectator, sourceDataTypes);
    CustomizedView customizedView = new CustomizedView(TEST_DB);
    customizedView.setState("p1", "h1", "testState");
    // Clear the flag before writing to the Customized View Path
    customizedViewChangeCalled.getAndSet(false);
    String customizedViewPath = PropertyPathBuilder.customizedView(CLUSTER_NAME, "typeA", TEST_DB);
    _spectator.getHelixDataAccessor().getBaseDataAccessor().set(customizedViewPath, customizedView.getRecord(), AccessOption.PERSISTENT);
    boolean onCustomizedViewChangeCalled = TestHelper.verify(() -> customizedViewChangeCalled.get(), WAIT_DURATION);
    Assert.assertTrue(onCustomizedViewChangeCalled);
    _spectator.getHelixDataAccessor().getBaseDataAccessor().remove(customizedViewPath, AccessOption.PERSISTENT);
    routingTableProvider.shutdown();
}

Example 2 with CustomizedView

use of org.apache.helix.model.CustomizedView in project helix by apache.

the class TestRoutingTableProvider method testGetRoutingTableSnapshot.

@Test(dependsOnMethods = "testCustomizedViewCorrectConstructor")
public void testGetRoutingTableSnapshot() throws Exception {
    Map<PropertyType, List<String>> sourceDataTypes = new HashMap<>();
    sourceDataTypes.put(PropertyType.CUSTOMIZEDVIEW, Arrays.asList("typeA", "typeB"));
    sourceDataTypes.put(PropertyType.EXTERNALVIEW, Collections.emptyList());
    RoutingTableProvider routingTableProvider = new RoutingTableProvider(_spectator, sourceDataTypes);
    CustomizedView customizedView1 = new CustomizedView("Resource1");
    customizedView1.setState("p1", "h1", "testState1");
    customizedView1.setState("p1", "h2", "testState1");
    customizedView1.setState("p2", "h1", "testState2");
    customizedView1.setState("p3", "h2", "testState3");
    String customizedViewPath1 = PropertyPathBuilder.customizedView(CLUSTER_NAME, "typeA", "Resource1");
    CustomizedView customizedView2 = new CustomizedView("Resource2");
    customizedView2.setState("p1", "h3", "testState3");
    customizedView2.setState("p1", "h4", "testState2");
    String customizedViewPath2 = PropertyPathBuilder.customizedView(CLUSTER_NAME, "typeB", "Resource2");
    _spectator.getHelixDataAccessor().getBaseDataAccessor().set(customizedViewPath1, customizedView1.getRecord(), AccessOption.PERSISTENT);
    _spectator.getHelixDataAccessor().getBaseDataAccessor().set(customizedViewPath2, customizedView2.getRecord(), AccessOption.PERSISTENT);
    RoutingTableSnapshot routingTableSnapshot = routingTableProvider.getRoutingTableSnapshot(PropertyType.CUSTOMIZEDVIEW, "typeA");
    Assert.assertEquals(routingTableSnapshot.getPropertyType(), PropertyType.CUSTOMIZEDVIEW);
    Assert.assertEquals(routingTableSnapshot.getCustomizedStateType(), "typeA");
    routingTableSnapshot = routingTableProvider.getRoutingTableSnapshot(PropertyType.CUSTOMIZEDVIEW, "typeB");
    Assert.assertEquals(routingTableSnapshot.getPropertyType(), PropertyType.CUSTOMIZEDVIEW);
    Assert.assertEquals(routingTableSnapshot.getCustomizedStateType(), "typeB");
    // Make sure snapshot information is correct
    // Check resources are in a correct state
    boolean isRoutingTableUpdatedProperly = TestHelper.verify(() -> {
        Map<String, Map<String, RoutingTableSnapshot>> routingTableSnapshots = routingTableProvider.getRoutingTableSnapshots();
        RoutingTableSnapshot routingTableSnapshotTypeA = routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).get("typeA");
        RoutingTableSnapshot routingTableSnapshotTypeB = routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).get("typeB");
        String typeAp1h1 = "noState";
        String typeAp1h2 = "noState";
        String typeAp2h1 = "noState";
        String typeAp3h2 = "noState";
        String typeBp1h2 = "noState";
        String typeBp1h4 = "noState";
        try {
            typeAp1h1 = routingTableSnapshotTypeA.getCustomizeViews().iterator().next().getStateMap("p1").get("h1");
            typeAp1h2 = routingTableSnapshotTypeA.getCustomizeViews().iterator().next().getStateMap("p1").get("h2");
            typeAp2h1 = routingTableSnapshotTypeA.getCustomizeViews().iterator().next().getStateMap("p2").get("h1");
            typeAp3h2 = routingTableSnapshotTypeA.getCustomizeViews().iterator().next().getStateMap("p3").get("h2");
            typeBp1h2 = routingTableSnapshotTypeB.getCustomizeViews().iterator().next().getStateMap("p1").get("h3");
            typeBp1h4 = routingTableSnapshotTypeB.getCustomizeViews().iterator().next().getStateMap("p1").get("h4");
        } catch (Exception e) {
            // ok because RoutingTable has not been updated yet
            return false;
        }
        return (routingTableSnapshots.size() == 2 && routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).size() == 2 && typeAp1h1.equals("testState1") && typeAp1h2.equals("testState1") && typeAp2h1.equals("testState2") && typeAp3h2.equals("testState3") && typeBp1h2.equals("testState3") && typeBp1h4.equals("testState2"));
    }, WAIT_DURATION);
    Assert.assertTrue(isRoutingTableUpdatedProperly, "RoutingTable has been updated properly");
    routingTableProvider.shutdown();
}

Example 3 with CustomizedView

use of org.apache.helix.model.CustomizedView in project helix by apache.

the class CustomizedViewAggregationStage method reportLatency.

private void reportLatency(ClusterEvent event, List<CustomizedView> updatedCustomizedViews, Map<String, CustomizedView> curCustomizedViews, Map<String, Map<Partition, Map<String, Long>>> updatedStartTimestamps, boolean[] updateSuccess) {
    ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
    if (clusterStatusMonitor == null) {
        return;
    }
    long curTime = System.currentTimeMillis();
    String clusterName = event.getClusterName();
    CustomizedViewMonitor customizedViewMonitor = clusterStatusMonitor.getOrCreateCustomizedViewMonitor(clusterName);
    for (int i = 0; i < updatedCustomizedViews.size(); i++) {
        CustomizedView newCV = updatedCustomizedViews.get(i);
        String resourceName = newCV.getResourceName();
        if (!updateSuccess[i]) {
            LOG.warn("Customized views are not updated successfully for resource {} on cluster {}", resourceName, clusterName);
            continue;
        }
        CustomizedView oldCV = curCustomizedViews.getOrDefault(resourceName, new CustomizedView(resourceName));
        Map<String, Map<String, String>> newPartitionStateMaps = newCV.getRecord().getMapFields();
        Map<String, Map<String, String>> oldPartitionStateMaps = oldCV.getRecord().getMapFields();
        Map<Partition, Map<String, Long>> partitionStartTimeMaps = updatedStartTimestamps.getOrDefault(resourceName, Collections.emptyMap());
        for (Map.Entry<String, Map<String, String>> partitionStateMapEntry : newPartitionStateMaps.entrySet()) {
            String partitionName = partitionStateMapEntry.getKey();
            Map<String, String> newStateMap = partitionStateMapEntry.getValue();
            Map<String, String> oldStateMap = oldPartitionStateMaps.getOrDefault(partitionName, Collections.emptyMap());
            if (!newStateMap.equals(oldStateMap)) {
                Map<String, Long> partitionStartTimeMap = partitionStartTimeMaps.getOrDefault(new Partition(partitionName), Collections.emptyMap());
                for (Map.Entry<String, String> stateMapEntry : newStateMap.entrySet()) {
                    String instanceName = stateMapEntry.getKey();
                    String newVal = stateMapEntry.getValue();
                    // So new value shouldn't be empty
                    if (!newVal.isEmpty() && !newVal.equals(oldStateMap.get(instanceName))) {
                        Long timestamp = partitionStartTimeMap.get(instanceName);
                        if (timestamp != null && timestamp > 0) {
                            customizedViewMonitor.recordUpdateToAggregationLatency(curTime - timestamp);
                        } else {
                            LOG.info("Failed to find customized state update time stamp for resource {} partition {}, instance {}, on cluster {} the number should be positive.", resourceName, partitionName, instanceName, clusterName);
                        }
                    }
                }
            }
        }
    }
}

Example 4 with CustomizedView

use of org.apache.helix.model.CustomizedView in project helix by apache.

the class ResourceControllerDataProvider method updateCustomizedViews.

/**
 * Update the cached customized view map
 * @param customizedViews
 */
public void updateCustomizedViews(String customizedStateType, List<CustomizedView> customizedViews) {
    if (!_customizedViewCacheMap.containsKey(customizedStateType)) {
        CustomizedViewCache customizedViewCache = new CustomizedViewCache(getClusterName(), customizedStateType);
        _customizedViewCacheMap.put(customizedStateType, customizedViewCache);
    }
    for (CustomizedView cv : customizedViews) {
        _customizedViewCacheMap.get(customizedStateType).getCustomizedViewCache().setProperty(cv);
    }
}

Example 5 with CustomizedView

use of org.apache.helix.model.CustomizedView in project helix by apache.

the class TestComputeAndCleanupCustomizedView method test.

@Test
public void test() throws Exception {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    int n = 2;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    2, // number of nodes
    n, // replicas
    2, "MasterSlave", // do rebalance
    true);
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    MockParticipantManager[] participants = new MockParticipantManager[n];
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    // add CUSTOMIZED_STATE_NAME2 to aggregation enabled types
    CustomizedStateConfig config = new CustomizedStateConfig();
    List<String> aggregationEnabledTypes = new ArrayList<>();
    aggregationEnabledTypes.add(CUSTOMIZED_STATE_NAME2);
    config.setAggregationEnabledTypes(aggregationEnabledTypes);
    accessor.setProperty(keyBuilder.customizedStateConfig(), config);
    // set INSTANCE1 to "STARTED" for CUSTOMIZED_STATE_NAME1
    CustomizedState customizedState = new CustomizedState(RESOURCE_NAME);
    customizedState.setState(PARTITION_NAME1, "STARTED");
    accessor.setProperty(keyBuilder.customizedState(INSTANCE_NAME1, CUSTOMIZED_STATE_NAME1, RESOURCE_NAME), customizedState);
    // verify the customized view is empty for CUSTOMIZED_STATE_NAME1
    Boolean result = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() {
            CustomizedView customizedView = accessor.getProperty(keyBuilder.customizedView(CUSTOMIZED_STATE_NAME1, RESOURCE_NAME));
            if (customizedView == null) {
                return true;
            }
            return false;
        }
    }, 12000);
    Thread.sleep(50);
    Assert.assertTrue(result, String.format("Customized view should not have state for" + " resource: %s, partition: %s", RESOURCE_NAME, PARTITION_NAME1));
    // add CUSTOMIZED_STATE_NAME1 to aggregation enabled types
    aggregationEnabledTypes.add(CUSTOMIZED_STATE_NAME1);
    config.setAggregationEnabledTypes(aggregationEnabledTypes);
    accessor.setProperty(keyBuilder.customizedStateConfig(), config);
    // verify the customized view should have "STARTED" for CUSTOMIZED_STATE_NAME1 for INSTANCE1,
    // but no state for INSTANCE2
    result = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() {
            CustomizedView customizedView = accessor.getProperty(keyBuilder.customizedView(CUSTOMIZED_STATE_NAME1, RESOURCE_NAME));
            if (customizedView != null) {
                Map<String, String> stateMap = customizedView.getRecord().getMapField(PARTITION_NAME1);
                return (stateMap.get(INSTANCE_NAME1).equals("STARTED"));
            }
            return false;
        }
    }, 12000);
    Thread.sleep(50);
    Assert.assertTrue(result, String.format("Customized view should have the state as STARTED for" + " instance: %s," + " resource: %s, partition: %s and state: %s", INSTANCE_NAME1, RESOURCE_NAME, PARTITION_NAME1, CUSTOMIZED_STATE_NAME1));
    result = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() {
            CustomizedView customizedView = accessor.getProperty(keyBuilder.customizedView(CUSTOMIZED_STATE_NAME1, RESOURCE_NAME));
            if (customizedView != null) {
                Map<String, String> stateMap = customizedView.getRecord().getMapField(PARTITION_NAME1);
                return !stateMap.containsKey(INSTANCE_NAME2);
            }
            return false;
        }
    }, 12000);
    Thread.sleep(50);
    Assert.assertTrue(result, String.format("Customized view should not have state for instance: " + "%s", INSTANCE_NAME2));
    // set INSTANCE2 to "STARTED" for CUSTOMIZED_STATE_NAME1
    customizedState = new CustomizedState(RESOURCE_NAME);
    customizedState.setState(PARTITION_NAME1, "STARTED");
    accessor.setProperty(keyBuilder.customizedState(INSTANCE_NAME2, CUSTOMIZED_STATE_NAME1, RESOURCE_NAME), customizedState);
    result = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() {
            CustomizedView customizedView = accessor.getProperty(keyBuilder.customizedView(CUSTOMIZED_STATE_NAME1, RESOURCE_NAME));
            if (customizedView != null) {
                Map<String, String> stateMap = customizedView.getRecord().getMapField(PARTITION_NAME1);
                if (stateMap.containsKey(INSTANCE_NAME2)) {
                    return (stateMap.get(INSTANCE_NAME1).equals("STARTED") && stateMap.get(INSTANCE_NAME2).equals("STARTED"));
                }
            }
            return false;
        }
    }, 12000);
    Thread.sleep(50);
    Assert.assertTrue(result, String.format("Customized view should have both instances state " + "as STARTED for" + " resource: %s, partition: %s and state: %s", RESOURCE_NAME, PARTITION_NAME1, CUSTOMIZED_STATE_NAME1));
    // set INSTANCE2 to "STARTED" for CUSTOMIZED_STATE_NAME2
    customizedState = new CustomizedState(RESOURCE_NAME);
    customizedState.setState(PARTITION_NAME2, "STARTED");
    accessor.setProperty(keyBuilder.customizedState(INSTANCE_NAME2, CUSTOMIZED_STATE_NAME2, RESOURCE_NAME), customizedState);
    result = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() {
            CustomizedView customizedView = accessor.getProperty(keyBuilder.customizedView(CUSTOMIZED_STATE_NAME2, RESOURCE_NAME));
            if (customizedView != null) {
                Map<String, String> stateMap = customizedView.getRecord().getMapField(PARTITION_NAME2);
                return (stateMap.get(INSTANCE_NAME2).equals("STARTED"));
            }
            return false;
        }
    }, 12000);
    Thread.sleep(50);
    Assert.assertTrue(result, String.format("Customized view should have state " + "as STARTED " + "for instance: %s, " + " resource: %s, partition: %s and state: %s", INSTANCE_NAME2, RESOURCE_NAME, PARTITION_NAME2, CUSTOMIZED_STATE_NAME2));
    // remove CUSTOMIZED_STATE_NAME1 from aggregation enabled types
    aggregationEnabledTypes.remove(CUSTOMIZED_STATE_NAME1);
    config.setAggregationEnabledTypes(aggregationEnabledTypes);
    accessor.setProperty(keyBuilder.customizedStateConfig(), config);
    result = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() {
            CustomizedView customizedView = accessor.getProperty(keyBuilder.customizedView(CUSTOMIZED_STATE_NAME1, RESOURCE_NAME));
            if (customizedView == null) {
                return true;
            }
            return false;
        }
    }, 12000);
    Thread.sleep(50);
    Assert.assertTrue(result, String.format("Customized view should not have state %s", CUSTOMIZED_STATE_NAME1));
    // disable controller
    ZKHelixAdmin admin = new ZKHelixAdmin(_gZkClient);
    admin.enableCluster(clusterName, false);
    ZkTestHelper.tryWaitZkEventsCleaned(controller.getZkClient());
    // drop resource
    admin.dropResource(clusterName, RESOURCE_NAME);
    // delete customized state manually, controller shall remove customized view when cluster is
    // enabled again
    accessor.removeProperty(keyBuilder.customizedState(INSTANCE_NAME1, CUSTOMIZED_STATE_NAME1, RESOURCE_NAME));
    accessor.removeProperty(keyBuilder.currentState(INSTANCE_NAME2, CUSTOMIZED_STATE_NAME1, RESOURCE_NAME));
    // re-enable controller shall remove orphan external view
    // System.out.println("re-enabling controller");
    admin.enableCluster(clusterName, true);
    result = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() {
            CustomizedView customizedView = accessor.getProperty(keyBuilder.customizedView(CUSTOMIZED_STATE_NAME1));
            if (customizedView == null) {
                return true;
            }
            return false;
        }
    }, 12000);
    Thread.sleep(50);
    Assert.assertTrue(result, String.format("customized view for should be null for  resource: %s, partition: %s and state: %s", RESOURCE_NAME, PARTITION_NAME1, CUSTOMIZED_STATE_NAME1));
    // clean up
    controller.syncStop();
    for (int i = 0; i < n; i++) {
        participants[i].syncStop();
    }
    TestHelper.dropCluster(clusterName, _gZkClient);
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}