Examples with IdealState org.apache.helix.model.IdealState used on opensource projects

Example 66 with IdealState

use of org.apache.helix.model.IdealState in project helix by apache.

the class TestZNRecordSizeLimit method testZNRecordSizeLimitUseZNRecordSerializer.

@Test
public void testZNRecordSizeLimitUseZNRecordSerializer() {
    String className = getShortClassName();
    System.out.println("START testZNRecordSizeLimitUseZNRecordSerializer at " + new Date(System.currentTimeMillis()));
    ZNRecordSerializer serializer = new ZNRecordSerializer();
    ZkClient zkClient = new ZkClient(ZK_ADDR);
    zkClient.setZkSerializer(serializer);
    String root = className;
    byte[] buf = new byte[1024];
    for (int i = 0; i < 1024; i++) {
        buf[i] = 'a';
    }
    String bufStr = new String(buf);
    // test zkClient
    // legal-sized data gets written to zk
    // write a znode of size less than 1m
    final ZNRecord smallRecord = new ZNRecord("normalsize");
    smallRecord.getSimpleFields().clear();
    for (int i = 0; i < 900; i++) {
        smallRecord.setSimpleField(i + "", bufStr);
    }
    String path1 = "/" + root + "/test1";
    zkClient.createPersistent(path1, true);
    zkClient.writeData(path1, smallRecord);
    ZNRecord record = zkClient.readData(path1);
    Assert.assertTrue(serializer.serialize(record).length > 900 * 1024);
    // oversized data doesn't create any data on zk
    // prepare a znode of size larger than 1m
    final ZNRecord largeRecord = new ZNRecord("oversize");
    largeRecord.getSimpleFields().clear();
    for (int i = 0; i < 1024; i++) {
        largeRecord.setSimpleField(i + "", bufStr);
    }
    String path2 = "/" + root + "/test2";
    zkClient.createPersistent(path2, true);
    try {
        zkClient.writeData(path2, largeRecord);
    } catch (HelixException e) {
        Assert.fail("Should not fail because data size is larger than 1M since compression applied");
    }
    record = zkClient.readData(path2);
    Assert.assertNotNull(record);
    // oversized write doesn't overwrite existing data on zk
    record = zkClient.readData(path1);
    try {
        zkClient.writeData(path1, largeRecord);
    } catch (HelixException e) {
        Assert.fail("Should not fail because data size is larger than 1M since compression applied");
    }
    ZNRecord recordNew = zkClient.readData(path1);
    byte[] arr = serializer.serialize(record);
    byte[] arrNew = serializer.serialize(recordNew);
    Assert.assertFalse(Arrays.equals(arr, arrNew));
    // test ZkDataAccessor
    ZKHelixAdmin admin = new ZKHelixAdmin(zkClient);
    admin.addCluster(className, true);
    InstanceConfig instanceConfig = new InstanceConfig("localhost_12918");
    admin.addInstance(className, instanceConfig);
    // oversized data should not create any new data on zk
    ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(className, new ZkBaseDataAccessor(zkClient));
    Builder keyBuilder = accessor.keyBuilder();
    IdealState idealState = new IdealState("currentState");
    idealState.setStateModelDefRef("MasterSlave");
    idealState.setRebalanceMode(RebalanceMode.SEMI_AUTO);
    idealState.setNumPartitions(10);
    for (int i = 0; i < 1024; i++) {
        idealState.getRecord().setSimpleField(i + "", bufStr);
    }
    boolean succeed = accessor.setProperty(keyBuilder.idealStates("TestDB0"), idealState);
    Assert.assertTrue(succeed);
    HelixProperty property = accessor.getProperty(keyBuilder.stateTransitionStatus("localhost_12918", "session_1", "partition_1"));
    Assert.assertNull(property);
    // legal sized data gets written to zk
    idealState.getRecord().getSimpleFields().clear();
    idealState.setStateModelDefRef("MasterSlave");
    idealState.setRebalanceMode(RebalanceMode.SEMI_AUTO);
    idealState.setNumPartitions(10);
    for (int i = 0; i < 900; i++) {
        idealState.getRecord().setSimpleField(i + "", bufStr);
    }
    succeed = accessor.setProperty(keyBuilder.idealStates("TestDB1"), idealState);
    Assert.assertTrue(succeed);
    record = accessor.getProperty(keyBuilder.idealStates("TestDB1")).getRecord();
    Assert.assertTrue(serializer.serialize(record).length > 900 * 1024);
    // oversized data should not update existing data on zk
    idealState.getRecord().getSimpleFields().clear();
    idealState.setStateModelDefRef("MasterSlave");
    idealState.setRebalanceMode(RebalanceMode.SEMI_AUTO);
    idealState.setNumPartitions(10);
    for (int i = 900; i < 1024; i++) {
        idealState.getRecord().setSimpleField(i + "", bufStr);
    }
    // System.out.println("record: " + idealState.getRecord());
    succeed = accessor.updateProperty(keyBuilder.idealStates("TestDB1"), idealState);
    Assert.assertTrue(succeed);
    recordNew = accessor.getProperty(keyBuilder.idealStates("TestDB1")).getRecord();
    arr = serializer.serialize(record);
    arrNew = serializer.serialize(recordNew);
    Assert.assertFalse(Arrays.equals(arr, arrNew));
    System.out.println("END testZNRecordSizeLimitUseZNRecordSerializer at " + new Date(System.currentTimeMillis()));
}

Example 67 with IdealState

use of org.apache.helix.model.IdealState in project helix by apache.

the class TestBatchMessageModeConfigs method setupResource.

private void setupResource(String dbName) throws InterruptedException {
    IdealState idealState = new FullAutoModeISBuilder(dbName).setStateModel("OnlineOffline").setStateModelFactoryName("TestFactory").setNumPartitions(10).setNumReplica(1).build();
    _setupTool.getClusterManagementTool().addResource(CLUSTER_NAME, dbName, idealState);
}

Example 68 with IdealState

use of org.apache.helix.model.IdealState in project helix by apache.

the class TestResourceThreadpoolSize method TestPerStateTransitionTypeThreadPool.

@Test
public void TestPerStateTransitionTypeThreadPool() throws InterruptedException {
    String MASTER_SLAVE = "MasterSlave";
    int customizedPoolSize = 22;
    for (MockParticipantManager participant : _participants) {
        participant.getStateMachineEngine().registerStateModelFactory(MASTER_SLAVE, new TestMasterSlaveStateModelFactory(customizedPoolSize), TEST_FACTORY);
    }
    // add db with customized thread pool
    IdealState idealState = new FullAutoModeISBuilder(WorkflowGenerator.DEFAULT_TGT_DB + "4").setStateModel(MASTER_SLAVE).setStateModelFactoryName(TEST_FACTORY).setNumPartitions(10).setNumReplica(1).build();
    _setupTool.getClusterManagementTool().addResource(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB + "4", idealState);
    _setupTool.rebalanceStorageCluster(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB + "4", 1);
    Thread.sleep(2000);
    // Verify OFFLINE -> SLAVE and SLAVE -> MASTER have different threadpool size.
    for (int i = 0; i < NODE_NR; i++) {
        DefaultMessagingService svc = (DefaultMessagingService) (_participants[i].getMessagingService());
        HelixTaskExecutor helixExecutor = svc.getExecutor();
        ThreadPoolExecutor executorOfflineToSlave = (ThreadPoolExecutor) (helixExecutor._executorMap.get(MessageType.STATE_TRANSITION + "." + WorkflowGenerator.DEFAULT_TGT_DB + "4" + "." + OFFLINE_TO_SLAVE));
        Assert.assertEquals(customizedPoolSize, executorOfflineToSlave.getMaximumPoolSize());
        ThreadPoolExecutor executorSlaveToMaster = (ThreadPoolExecutor) (helixExecutor._executorMap.get(MessageType.STATE_TRANSITION + "." + WorkflowGenerator.DEFAULT_TGT_DB + "4" + "." + SLAVE_TO_MASTER));
        Assert.assertEquals(customizedPoolSize + 5, executorSlaveToMaster.getMaximumPoolSize());
    }
}

Example 69 with IdealState

use of org.apache.helix.model.IdealState in project helix by apache.

the class TestResourceThreadpoolSize method TestCustomizedResourceThreadPool.

@Test
public void TestCustomizedResourceThreadPool() {
    int customizedPoolSize = 7;
    int configuredPoolSize = 9;
    for (MockParticipantManager participant : _participants) {
        participant.getStateMachineEngine().registerStateModelFactory(ONLINE_OFFLINE, new TestOnlineOfflineStateModelFactory(customizedPoolSize, 0), TEST_FACTORY);
    }
    // add db with default thread pool
    _setupTool.addResourceToCluster(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB + "1", 64, STATE_MODEL);
    _setupTool.rebalanceStorageCluster(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB + "1", 3);
    // add db with customized thread pool
    IdealState idealState = new FullAutoModeISBuilder(WorkflowGenerator.DEFAULT_TGT_DB + "2").setStateModel(ONLINE_OFFLINE).setStateModelFactoryName(TEST_FACTORY).setNumPartitions(10).setNumReplica(1).build();
    _setupTool.getClusterManagementTool().addResource(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB + "2", idealState);
    _setupTool.rebalanceStorageCluster(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB + "2", 1);
    // add db with configured pool size
    idealState = new FullAutoModeISBuilder(WorkflowGenerator.DEFAULT_TGT_DB + "3").setStateModel(ONLINE_OFFLINE).setStateModelFactoryName(TEST_FACTORY).setNumPartitions(10).setNumReplica(1).build();
    _setupTool.getClusterManagementTool().addResource(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB + "3", idealState);
    setResourceThreadPoolSize(WorkflowGenerator.DEFAULT_TGT_DB + "3", configuredPoolSize);
    _setupTool.rebalanceStorageCluster(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB + "3", 1);
    boolean result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME));
    Assert.assertTrue(result);
    for (int i = 0; i < NODE_NR; i++) {
        DefaultMessagingService svc = (DefaultMessagingService) (_participants[i].getMessagingService());
        HelixTaskExecutor helixExecutor = svc.getExecutor();
        ThreadPoolExecutor executor = (ThreadPoolExecutor) (helixExecutor._executorMap.get(MessageType.STATE_TRANSITION + "." + WorkflowGenerator.DEFAULT_TGT_DB + "1"));
        Assert.assertNull(executor);
        executor = (ThreadPoolExecutor) (helixExecutor._executorMap.get(MessageType.STATE_TRANSITION + "." + WorkflowGenerator.DEFAULT_TGT_DB + "2"));
        Assert.assertEquals(customizedPoolSize, executor.getMaximumPoolSize());
        executor = (ThreadPoolExecutor) (helixExecutor._executorMap.get(MessageType.STATE_TRANSITION + "." + WorkflowGenerator.DEFAULT_TGT_DB + "3"));
        Assert.assertEquals(configuredPoolSize, executor.getMaximumPoolSize());
    }
}

Example 70 with IdealState

use of org.apache.helix.model.IdealState in project helix by apache.

the class TestResourceMonitor method testReportData.

@Test()
public void testReportData() throws JMException {
    final int n = 5;
    ResourceMonitor monitor = new ResourceMonitor(_clusterName, _dbName, new ObjectName("testDomain:key=value"));
    monitor.register();
    List<String> instances = new ArrayList<String>();
    for (int i = 0; i < n; i++) {
        String instance = "localhost_" + (12918 + i);
        instances.add(instance);
    }
    ZNRecord idealStateRecord = DefaultIdealStateCalculator.calculateIdealState(instances, _partitions, _replicas - 1, _dbName, "MASTER", "SLAVE");
    IdealState idealState = new IdealState(deepCopyZNRecord(idealStateRecord));
    idealState.setMinActiveReplicas(_replicas - 1);
    ExternalView externalView = new ExternalView(deepCopyZNRecord(idealStateRecord));
    StateModelDefinition stateModelDef = BuiltInStateModelDefinitions.MasterSlave.getStateModelDefinition();
    monitor.updateResource(externalView, idealState, stateModelDef);
    Assert.assertEquals(monitor.getDifferenceWithIdealStateGauge(), 0);
    Assert.assertEquals(monitor.getErrorPartitionGauge(), 0);
    Assert.assertEquals(monitor.getExternalViewPartitionGauge(), _partitions);
    Assert.assertEquals(monitor.getPartitionGauge(), _partitions);
    Assert.assertEquals(monitor.getMissingMinActiveReplicaPartitionGauge(), 0);
    Assert.assertEquals(monitor.getMissingReplicaPartitionGauge(), 0);
    Assert.assertEquals(monitor.getMissingTopStatePartitionGauge(), 0);
    Assert.assertEquals(monitor.getBeanName(), _clusterName + " " + _dbName);
    int errorCount = 5;
    Random r = new Random();
    int start = r.nextInt(_partitions - errorCount - 1);
    for (int i = start; i < start + errorCount; i++) {
        String partition = _dbName + "_" + i;
        Map<String, String> map = externalView.getStateMap(partition);
        for (String key : map.keySet()) {
            if (map.get(key).equalsIgnoreCase("SLAVE")) {
                map.put(key, "ERROR");
                break;
            }
        }
        externalView.setStateMap(partition, map);
    }
    monitor.updateResource(externalView, idealState, stateModelDef);
    Assert.assertEquals(monitor.getDifferenceWithIdealStateGauge(), errorCount);
    Assert.assertEquals(monitor.getErrorPartitionGauge(), errorCount);
    Assert.assertEquals(monitor.getExternalViewPartitionGauge(), _partitions);
    Assert.assertEquals(monitor.getPartitionGauge(), _partitions);
    Assert.assertEquals(monitor.getMissingMinActiveReplicaPartitionGauge(), 0);
    Assert.assertEquals(monitor.getMissingReplicaPartitionGauge(), errorCount);
    Assert.assertEquals(monitor.getMissingTopStatePartitionGauge(), 0);
    int lessMinActiveReplica = 6;
    externalView = new ExternalView(deepCopyZNRecord(idealStateRecord));
    start = r.nextInt(_partitions - lessMinActiveReplica - 1);
    for (int i = start; i < start + lessMinActiveReplica; i++) {
        String partition = _dbName + "_" + i;
        Map<String, String> map = externalView.getStateMap(partition);
        Iterator<String> it = map.keySet().iterator();
        int flag = 0;
        while (it.hasNext()) {
            String key = it.next();
            if (map.get(key).equalsIgnoreCase("SLAVE")) {
                if (flag++ % 2 == 0) {
                    map.put(key, "OFFLINE");
                } else {
                    it.remove();
                }
            }
        }
        externalView.setStateMap(partition, map);
    }
    monitor.updateResource(externalView, idealState, stateModelDef);
    Assert.assertEquals(monitor.getDifferenceWithIdealStateGauge(), lessMinActiveReplica);
    Assert.assertEquals(monitor.getErrorPartitionGauge(), 0);
    Assert.assertEquals(monitor.getExternalViewPartitionGauge(), _partitions);
    Assert.assertEquals(monitor.getPartitionGauge(), _partitions);
    Assert.assertEquals(monitor.getMissingMinActiveReplicaPartitionGauge(), lessMinActiveReplica);
    Assert.assertEquals(monitor.getMissingReplicaPartitionGauge(), lessMinActiveReplica);
    Assert.assertEquals(monitor.getMissingTopStatePartitionGauge(), 0);
    int lessReplica = 4;
    externalView = new ExternalView(deepCopyZNRecord(idealStateRecord));
    start = r.nextInt(_partitions - lessReplica - 1);
    for (int i = start; i < start + lessReplica; i++) {
        String partition = _dbName + "_" + i;
        Map<String, String> map = externalView.getStateMap(partition);
        int flag = 0;
        Iterator<String> it = map.keySet().iterator();
        while (it.hasNext()) {
            String key = it.next();
            if (map.get(key).equalsIgnoreCase("SLAVE")) {
                if (flag++ % 2 == 0) {
                    map.put(key, "OFFLINE");
                } else {
                    it.remove();
                }
                break;
            }
        }
        externalView.setStateMap(partition, map);
    }
    monitor.updateResource(externalView, idealState, stateModelDef);
    Assert.assertEquals(monitor.getDifferenceWithIdealStateGauge(), lessReplica);
    Assert.assertEquals(monitor.getErrorPartitionGauge(), 0);
    Assert.assertEquals(monitor.getExternalViewPartitionGauge(), _partitions);
    Assert.assertEquals(monitor.getPartitionGauge(), _partitions);
    Assert.assertEquals(monitor.getMissingMinActiveReplicaPartitionGauge(), 0);
    Assert.assertEquals(monitor.getMissingReplicaPartitionGauge(), lessReplica);
    Assert.assertEquals(monitor.getMissingTopStatePartitionGauge(), 0);
    int missTopState = 7;
    externalView = new ExternalView(deepCopyZNRecord(idealStateRecord));
    start = r.nextInt(_partitions - missTopState - 1);
    for (int i = start; i < start + missTopState; i++) {
        String partition = _dbName + "_" + i;
        Map<String, String> map = externalView.getStateMap(partition);
        int flag = 0;
        for (String key : map.keySet()) {
            if (map.get(key).equalsIgnoreCase("MASTER")) {
                if (flag++ % 2 == 0) {
                    map.put(key, "OFFLINE");
                } else {
                    map.remove(key);
                }
                break;
            }
        }
        externalView.setStateMap(partition, map);
    }
    monitor.updateResource(externalView, idealState, stateModelDef);
    Assert.assertEquals(monitor.getDifferenceWithIdealStateGauge(), missTopState);
    Assert.assertEquals(monitor.getErrorPartitionGauge(), 0);
    Assert.assertEquals(monitor.getExternalViewPartitionGauge(), _partitions);
    Assert.assertEquals(monitor.getPartitionGauge(), _partitions);
    Assert.assertEquals(monitor.getMissingMinActiveReplicaPartitionGauge(), 0);
    Assert.assertEquals(monitor.getMissingReplicaPartitionGauge(), missTopState);
    Assert.assertEquals(monitor.getMissingTopStatePartitionGauge(), missTopState);
}