Examples with HelixDataAccessor org.apache.helix.HelixDataAccessor used on opensource projects

Example 36 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project helix by apache.

the class TestZkFlapping method testParticipantFlapping.

@Test
public void testParticipantFlapping() throws Exception {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    final HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    final PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    32, // number of nodes
    1, // replicas
    1, "MasterSlave", false);
    final String instanceName = "localhost_12918";
    MockParticipantManager participant = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
    participant.syncStart();
    final ZkClient client = participant.getZkClient();
    final ZkStateCountListener listener = new ZkStateCountListener();
    client.subscribeStateChanges(listener);
    final AtomicInteger expectDisconnectCnt = new AtomicInteger(0);
    final int n = ZKHelixManager.MAX_DISCONNECT_THRESHOLD;
    for (int i = 0; i < n; i++) {
        String oldSessionId = ZkTestHelper.getSessionId(client);
        ZkTestHelper.simulateZkStateDisconnected(client);
        expectDisconnectCnt.incrementAndGet();
        // wait until we get invoked by zk state change to disconnected
        TestHelper.verify(new Verifier() {

            @Override
            public boolean verify() throws Exception {
                return listener.count == expectDisconnectCnt.get();
            }
        }, 30 * 1000);
        String newSessionId = ZkTestHelper.getSessionId(client);
        Assert.assertEquals(newSessionId, oldSessionId);
    }
    client.unsubscribeStateChanges(listener);
    // make sure participant is NOT disconnected
    LiveInstance liveInstance = accessor.getProperty(keyBuilder.liveInstance(instanceName));
    Assert.assertNotNull(liveInstance, "Live-instance should exist after " + n + " disconnects");
    // trigger flapping
    ZkTestHelper.simulateZkStateDisconnected(client);
    // wait until we get invoked by zk state change to disconnected
    boolean success = TestHelper.verify(new Verifier() {

        @Override
        public boolean verify() throws Exception {
            return client.getShutdownTrigger();
        }
    }, 30 * 1000);
    Assert.assertTrue(success, "The " + (n + 1) + "th disconnect event should trigger ZkHelixManager#disonnect");
    // make sure participant is disconnected
    success = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() throws Exception {
            LiveInstance liveInstance = accessor.getProperty(keyBuilder.liveInstance(instanceName));
            return liveInstance == null;
        }
    }, 3 * 1000);
    Assert.assertTrue(success, "Live-instance should be gone after " + (n + 1) + " disconnects");
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 37 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project helix by apache.

the class TestZkHelixAdmin method testAddRemoveMsgConstraint.

// test add/remove message constraint
@Test
public void testAddRemoveMsgConstraint() {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    HelixAdmin tool = new ZKHelixAdmin(_gZkClient);
    tool.addCluster(clusterName, true);
    Assert.assertTrue(ZKUtil.isClusterSetup(clusterName, _gZkClient), "Cluster should be setup");
    // test admin.getMessageConstraints()
    ClusterConstraints constraints = tool.getConstraints(clusterName, ConstraintType.MESSAGE_CONSTRAINT);
    Assert.assertNull(constraints, "message-constraint should NOT exist for cluster: " + className);
    // remove non-exist constraint
    try {
        tool.removeConstraint(clusterName, ConstraintType.MESSAGE_CONSTRAINT, "constraint1");
    // will leave a null message-constraint znode on zk
    } catch (Exception e) {
        Assert.fail("Should not throw exception when remove a non-exist constraint.");
    }
    // add a message constraint
    ConstraintItemBuilder builder = new ConstraintItemBuilder();
    builder.addConstraintAttribute(ConstraintAttribute.RESOURCE.toString(), "MyDB").addConstraintAttribute(ConstraintAttribute.CONSTRAINT_VALUE.toString(), "1");
    tool.setConstraint(clusterName, ConstraintType.MESSAGE_CONSTRAINT, "constraint1", builder.build());
    HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    PropertyKey.Builder keyBuilder = new PropertyKey.Builder(clusterName);
    constraints = accessor.getProperty(keyBuilder.constraint(ConstraintType.MESSAGE_CONSTRAINT.toString()));
    Assert.assertNotNull(constraints, "message-constraint should exist");
    ConstraintItem item = constraints.getConstraintItem("constraint1");
    Assert.assertNotNull(item, "message-constraint for constraint1 should exist");
    Assert.assertEquals(item.getConstraintValue(), "1");
    Assert.assertEquals(item.getAttributeValue(ConstraintAttribute.RESOURCE), "MyDB");
    // test admin.getMessageConstraints()
    constraints = tool.getConstraints(clusterName, ConstraintType.MESSAGE_CONSTRAINT);
    Assert.assertNotNull(constraints, "message-constraint should exist");
    item = constraints.getConstraintItem("constraint1");
    Assert.assertNotNull(item, "message-constraint for constraint1 should exist");
    Assert.assertEquals(item.getConstraintValue(), "1");
    Assert.assertEquals(item.getAttributeValue(ConstraintAttribute.RESOURCE), "MyDB");
    // remove a exist message-constraint
    tool.removeConstraint(clusterName, ConstraintType.MESSAGE_CONSTRAINT, "constraint1");
    constraints = accessor.getProperty(keyBuilder.constraint(ConstraintType.MESSAGE_CONSTRAINT.toString()));
    Assert.assertNotNull(constraints, "message-constraint should exist");
    item = constraints.getConstraintItem("constraint1");
    Assert.assertNull(item, "message-constraint for constraint1 should NOT exist");
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 38 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project helix by apache.

the class TestZKLiveInstanceData method testDataChange.

@Test
public void testDataChange() throws Exception {
    // Create an admin and add LiveInstanceChange listener to it
    HelixManager adminManager = HelixManagerFactory.getZKHelixManager(clusterName, null, InstanceType.ADMINISTRATOR, ZK_ADDR);
    adminManager.connect();
    final BlockingQueue<List<LiveInstance>> changeList = new LinkedBlockingQueue<List<LiveInstance>>();
    adminManager.addLiveInstanceChangeListener(new LiveInstanceChangeListener() {

        @Override
        public void onLiveInstanceChange(List<LiveInstance> liveInstances, NotificationContext changeContext) {
            // The queue is basically unbounded, so shouldn't throw exception when calling
            // "add".
            changeList.add(deepCopy(liveInstances));
        }
    });
    // Check the initial condition
    List<LiveInstance> instances = changeList.poll(1, TimeUnit.SECONDS);
    Assert.assertNotNull(instances, "Expecting a list of live instance");
    Assert.assertTrue(instances.isEmpty(), "Expecting an empty list of live instance");
    // Join as participant, should trigger a live instance change event
    HelixManager manager = HelixManagerFactory.getZKHelixManager(clusterName, "localhost_54321", InstanceType.PARTICIPANT, ZK_ADDR);
    manager.connect();
    instances = changeList.poll(1, TimeUnit.SECONDS);
    Assert.assertNotNull(instances, "Expecting a list of live instance");
    Assert.assertEquals(instances.size(), 1, "Expecting one live instance");
    Assert.assertEquals(instances.get(0).getInstanceName(), manager.getInstanceName());
    // Update data in the live instance node, should trigger another live instance change
    // event
    HelixDataAccessor helixDataAccessor = manager.getHelixDataAccessor();
    PropertyKey propertyKey = helixDataAccessor.keyBuilder().liveInstance(manager.getInstanceName());
    LiveInstance instance = helixDataAccessor.getProperty(propertyKey);
    Map<String, String> map = new TreeMap<String, String>();
    map.put("k1", "v1");
    instance.getRecord().setMapField("test", map);
    Assert.assertTrue(helixDataAccessor.updateProperty(propertyKey, instance), "Failed to update live instance node");
    instances = changeList.poll(1, TimeUnit.SECONDS);
    Assert.assertNotNull(instances, "Expecting a list of live instance");
    Assert.assertEquals(instances.get(0).getRecord().getMapField("test"), map, "Wrong map data.");
    manager.disconnect();
    // wait for callback finish
    Thread.sleep(1000);
    instances = changeList.poll(1, TimeUnit.SECONDS);
    Assert.assertNotNull(instances, "Expecting a list of live instance");
    Assert.assertTrue(instances.isEmpty(), "Expecting an empty list of live instance");
    adminManager.disconnect();
}

Example 39 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project helix by apache.

the class TestZKPathDataDumpTask method testCapacityReached.

@Test
public void testCapacityReached() throws Exception {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    int n = 1;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    2, // number of nodes
    n, // replicas
    1, "MasterSlave", // do rebalance
    true);
    HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    BaseDataAccessor<ZNRecord> baseAccessor = accessor.getBaseDataAccessor();
    HelixManager manager = mock(HelixManager.class);
    when(manager.getHelixDataAccessor()).thenReturn(accessor);
    when(manager.getClusterName()).thenReturn(clusterName);
    // run dump task without statusUpdates and errors, should not remove any existing
    // statusUpdate/error paths
    ZKPathDataDumpTask task = new ZKPathDataDumpTask(manager, Long.MAX_VALUE, Long.MAX_VALUE, 1);
    task.run();
    PropertyKey controllerStatusUpdateKey = keyBuilder.controllerTaskStatuses();
    Assert.assertTrue(baseAccessor.exists(controllerStatusUpdateKey.getPath(), 0));
    PropertyKey controllerErrorKey = keyBuilder.controllerTaskErrors();
    Assert.assertTrue(baseAccessor.exists(controllerErrorKey.getPath(), 0));
    PropertyKey statusUpdateKey = keyBuilder.stateTransitionStatus("localhost_12918");
    Assert.assertTrue(baseAccessor.exists(statusUpdateKey.getPath(), 0));
    PropertyKey errorKey = keyBuilder.stateTransitionErrors("localhost_12918");
    Assert.assertTrue(baseAccessor.exists(errorKey.getPath(), 0));
    // add participant status updates and errors
    statusUpdateKey = keyBuilder.stateTransitionStatus("localhost_12918", "session_0", "TestDB0", "TestDB0_0");
    accessor.setProperty(statusUpdateKey, new StatusUpdate(new ZNRecord("statusUpdate")));
    errorKey = keyBuilder.stateTransitionError("localhost_12918", "session_0", "TestDB0", "TestDB0_0");
    accessor.setProperty(errorKey, new Error(new ZNRecord("error")));
    // add controller status updates and errors (one of each, should not trigger anything)
    controllerStatusUpdateKey = keyBuilder.controllerTaskStatus("session_0", "TestDB");
    accessor.setProperty(controllerStatusUpdateKey, new StatusUpdate(new ZNRecord("controllerStatusUpdate")));
    controllerErrorKey = keyBuilder.controllerTaskError("TestDB_error");
    accessor.setProperty(controllerErrorKey, new Error(new ZNRecord("controllerError")));
    // run dump task, should not remove anything because the threshold is not exceeded
    task.run();
    Assert.assertTrue(baseAccessor.exists(controllerStatusUpdateKey.getPath(), 0));
    Assert.assertTrue(baseAccessor.exists(controllerErrorKey.getPath(), 0));
    Assert.assertTrue(baseAccessor.exists(statusUpdateKey.getPath(), 0));
    Assert.assertTrue(baseAccessor.exists(errorKey.getPath(), 0));
    // add a second set of all status updates and errors
    statusUpdateKey = keyBuilder.stateTransitionStatus("localhost_12918", "session_0", "TestDB0", "TestDB0_1");
    accessor.setProperty(statusUpdateKey, new StatusUpdate(new ZNRecord("statusUpdate")));
    errorKey = keyBuilder.stateTransitionError("localhost_12918", "session_0", "TestDB0", "TestDB0_1");
    accessor.setProperty(errorKey, new Error(new ZNRecord("error")));
    controllerStatusUpdateKey = keyBuilder.controllerTaskStatus("session_0", "TestDB1");
    accessor.setProperty(controllerStatusUpdateKey, new StatusUpdate(new ZNRecord("controllerStatusUpdate")));
    controllerErrorKey = keyBuilder.controllerTaskError("TestDB1_error");
    accessor.setProperty(controllerErrorKey, new Error(new ZNRecord("controllerError")));
    // run dump task, should remove everything since capacities are exceeded
    task.run();
    Assert.assertFalse(baseAccessor.exists(controllerStatusUpdateKey.getPath(), 0));
    Assert.assertFalse(baseAccessor.exists(controllerErrorKey.getPath(), 0));
    Assert.assertFalse(baseAccessor.exists(statusUpdateKey.getPath(), 0));
    Assert.assertFalse(baseAccessor.exists(errorKey.getPath(), 0));
}

Example 40 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project helix by apache.

the class TestTaskRebalancer method testNamedQueue.

@Test
public void testNamedQueue() throws Exception {
    String queueName = TestHelper.getTestMethodName();
    // Create a queue
    JobQueue queue = new JobQueue.Builder(queueName).build();
    _driver.createQueue(queue);
    // Enqueue jobs
    Set<String> master = Sets.newHashSet("MASTER");
    Set<String> slave = Sets.newHashSet("SLAVE");
    JobConfig.Builder job1 = new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND).setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB).setTargetPartitionStates(master);
    JobConfig.Builder job2 = new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND).setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB).setTargetPartitionStates(slave);
    _driver.enqueueJob(queueName, "masterJob", job1);
    _driver.enqueueJob(queueName, "slaveJob", job2);
    // Ensure successful completion
    String namespacedJob1 = queueName + "_masterJob";
    String namespacedJob2 = queueName + "_slaveJob";
    _driver.pollForJobState(queueName, namespacedJob1, TaskState.COMPLETED);
    _driver.pollForJobState(queueName, namespacedJob2, TaskState.COMPLETED);
    JobContext masterJobContext = _driver.getJobContext(namespacedJob1);
    JobContext slaveJobContext = _driver.getJobContext(namespacedJob2);
    // Ensure correct ordering
    long job1Finish = masterJobContext.getFinishTime();
    long job2Start = slaveJobContext.getStartTime();
    Assert.assertTrue(job2Start >= job1Finish);
    // Flush queue and check cleanup
    _driver.flushQueue(queueName);
    HelixDataAccessor accessor = _manager.getHelixDataAccessor();
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    Assert.assertNull(accessor.getProperty(keyBuilder.idealStates(namespacedJob1)));
    Assert.assertNull(accessor.getProperty(keyBuilder.resourceConfig(namespacedJob1)));
    Assert.assertNull(accessor.getProperty(keyBuilder.idealStates(namespacedJob2)));
    Assert.assertNull(accessor.getProperty(keyBuilder.resourceConfig(namespacedJob2)));
    WorkflowConfig workflowCfg = _driver.getWorkflowConfig(queueName);
    JobDag dag = workflowCfg.getJobDag();
    Assert.assertFalse(dag.getAllNodes().contains(namespacedJob1));
    Assert.assertFalse(dag.getAllNodes().contains(namespacedJob2));
    Assert.assertFalse(dag.getChildrenToParents().containsKey(namespacedJob1));
    Assert.assertFalse(dag.getChildrenToParents().containsKey(namespacedJob2));
    Assert.assertFalse(dag.getParentsToChildren().containsKey(namespacedJob1));
    Assert.assertFalse(dag.getParentsToChildren().containsKey(namespacedJob2));
}