Examples with DataUpdater org.apache.helix.zookeeper.zkclient.DataUpdater used on opensource projects

Example 11 with DataUpdater

use of org.apache.helix.zookeeper.zkclient.DataUpdater in project helix by apache.

the class CurrentStateCleanUp method cleanupCurrentStatesForCluster.

public static void cleanupCurrentStatesForCluster(String zkConnectString, String clusterName, String instanceName, String session) throws Exception {
    HelixManager manager = HelixManagerFactory.getZKHelixManager(clusterName, "Administrator", InstanceType.ADMINISTRATOR, zkConnectString);
    manager.connect();
    try {
        HelixDataAccessor accessor = manager.getHelixDataAccessor();
        DataUpdater<ZNRecord> updater = new DataUpdater<ZNRecord>() {

            @Override
            public ZNRecord update(ZNRecord currentData) {
                if (currentData == null) {
                    return null;
                }
                Set<String> partitionToRemove = new HashSet<>();
                for (String partition : currentData.getMapFields().keySet()) {
                    if (currentData.getMapField(partition).get("CURRENT_STATE").equals(HelixDefinedState.DROPPED.name())) {
                        partitionToRemove.add(partition);
                    }
                }
                currentData.getMapFields().keySet().removeAll(partitionToRemove);
                return currentData;
            }
        };
        LOG.info(String.format("Processing cleaning current state for instance: %s", instanceName));
        List<String> currentStateNames = accessor.getChildNames(accessor.keyBuilder().currentStates(instanceName, session));
        List<String> taskCurrentStateNames = accessor.getChildNames(accessor.keyBuilder().taskCurrentStates(instanceName, session));
        List<PropertyKey> allCurrentStateKeys = new ArrayList<>();
        currentStateNames.stream().map(name -> accessor.keyBuilder().currentState(instanceName, session, name)).forEach(allCurrentStateKeys::add);
        taskCurrentStateNames.stream().map(name -> accessor.keyBuilder().taskCurrentState(instanceName, session, name)).forEach(allCurrentStateKeys::add);
        List<String> pathsToRemove = new ArrayList<>();
        for (PropertyKey key : allCurrentStateKeys) {
            accessor.getBaseDataAccessor().update(key.getPath(), updater, AccessOption.PERSISTENT);
            CurrentState currentState = accessor.getProperty(key);
            if (currentState.getPartitionStateMap().size() == 0) {
                pathsToRemove.add(key.getPath());
                LOG.info(String.format("Remove current state for path %s", key.getPath()));
            }
        }
        accessor.getBaseDataAccessor().remove(pathsToRemove, AccessOption.PERSISTENT);
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        manager.disconnect();
    }
}

Example 12 with DataUpdater

use of org.apache.helix.zookeeper.zkclient.DataUpdater in project helix by apache.

the class TestZkBaseDataAccessor method testSyncUpdate.

@Test
public void testSyncUpdate() {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String testName = className + "_" + methodName;
    System.out.println("START " + testName + " at " + new Date(System.currentTimeMillis()));
    String path = String.format("/%s/%s", _rootPath, "msg_0");
    ZNRecord record = new ZNRecord("msg_0");
    ZkBaseDataAccessor<ZNRecord> accessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
    boolean success = accessor.update(path, new ZNRecordUpdater(record), AccessOption.PERSISTENT);
    Assert.assertTrue(success);
    ZNRecord getRecord = _gZkClient.readData(path);
    Assert.assertNotNull(getRecord);
    Assert.assertEquals(getRecord.getId(), "msg_0");
    record.setSimpleField("key0", "value0");
    success = accessor.update(path, new ZNRecordUpdater(record), AccessOption.PERSISTENT);
    Assert.assertTrue(success);
    getRecord = _gZkClient.readData(path);
    Assert.assertNotNull(getRecord);
    Assert.assertEquals(getRecord.getSimpleFields().size(), 1);
    Assert.assertNotNull(getRecord.getSimpleField("key0"));
    Assert.assertEquals(getRecord.getSimpleField("key0"), "value0");
    // test throw exception from updater
    success = accessor.update(path, new DataUpdater<ZNRecord>() {

        @Override
        public ZNRecord update(ZNRecord currentData) {
            throw new RuntimeException("IGNORABLE: test throw exception from updater");
        }
    }, AccessOption.PERSISTENT);
    Assert.assertFalse(success);
    getRecord = _gZkClient.readData(path);
    Assert.assertNotNull(getRecord);
    Assert.assertEquals(getRecord.getSimpleFields().size(), 1);
    System.out.println("END " + testName + " at " + new Date(System.currentTimeMillis()));
}

Example 13 with DataUpdater

use of org.apache.helix.zookeeper.zkclient.DataUpdater in project helix by apache.

the class TestZkBaseDataAccessor method testAsyncZkBaseDataAccessor.

@Test
public void testAsyncZkBaseDataAccessor() {
    System.out.println("START TestZkBaseDataAccessor.async at " + new Date(System.currentTimeMillis()));
    String root = _rootPath;
    _gZkClient.deleteRecursively("/" + root);
    ZkBaseDataAccessor<ZNRecord> accessor = new ZkBaseDataAccessor<>(_gZkClient);
    // test async createChildren
    List<ZNRecord> records = new ArrayList<>();
    List<String> paths = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        paths.add(PropertyPathBuilder.instanceMessage(root, "host_1", msgId));
        records.add(new ZNRecord(msgId));
    }
    boolean[] success = accessor.createChildren(paths, records, AccessOption.PERSISTENT);
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        Assert.assertTrue(success[i], "Should succeed in create " + msgId);
    }
    // test get what we created
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        String path = PropertyPathBuilder.instanceMessage(root, "host_1", msgId);
        ZNRecord record = _gZkClient.readData(path);
        Assert.assertEquals(record.getId(), msgId, "Should get what we created");
    }
    // test async createChildren with TTL
    System.setProperty("zookeeper.extendedTypesEnabled", "true");
    records = new ArrayList<>();
    paths = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        paths.add(PropertyPathBuilder.instanceMessage(root, "host_2", msgId));
        records.add(new ZNRecord(msgId));
    }
    success = accessor.createChildren(paths, records, AccessOption.PERSISTENT_WITH_TTL, 1L);
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        Assert.assertTrue(success[i], "Should succeed in create " + msgId);
    }
    // test get what we created
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        String path = PropertyPathBuilder.instanceMessage(root, "host_2", msgId);
        ZNRecord record = _gZkClient.readData(path);
        Assert.assertEquals(record.getId(), msgId, "Should get what we created");
    }
    // test async createChildren with Container mode
    records = new ArrayList<>();
    paths = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        paths.add(PropertyPathBuilder.instanceMessage(root, "host_3", msgId));
        records.add(new ZNRecord(msgId));
    }
    success = accessor.createChildren(paths, records, AccessOption.CONTAINER);
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        Assert.assertTrue(success[i], "Should succeed in create " + msgId);
    }
    // test get what we created
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        String path = PropertyPathBuilder.instanceMessage(root, "host_3", msgId);
        ZNRecord record = _gZkClient.readData(path);
        Assert.assertEquals(record.getId(), msgId, "Should get what we created");
    }
    System.clearProperty("zookeeper.extendedTypesEnabled");
    // test async setChildren
    records = new ArrayList<>();
    paths = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        paths.add(PropertyPathBuilder.instanceMessage(root, "host_1", msgId));
        ZNRecord newRecord = new ZNRecord(msgId);
        newRecord.setSimpleField("key1", "value1");
        records.add(newRecord);
    }
    success = accessor.setChildren(paths, records, AccessOption.PERSISTENT);
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        Assert.assertTrue(success[i], "Should succeed in set " + msgId);
    }
    // test get what we set
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        String path = PropertyPathBuilder.instanceMessage(root, "host_1", msgId);
        ZNRecord record = _gZkClient.readData(path);
        Assert.assertEquals(record.getSimpleFields().size(), 1, "Should have 1 simple field set");
        Assert.assertEquals(record.getSimpleField("key1"), "value1", "Should have value1 set");
    }
    // test async updateChildren
    // records = new ArrayList<ZNRecord>();
    List<DataUpdater<ZNRecord>> znrecordUpdaters = new ArrayList<DataUpdater<ZNRecord>>();
    paths = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        paths.add(PropertyPathBuilder.instanceMessage(root, "host_1", msgId));
        ZNRecord newRecord = new ZNRecord(msgId);
        newRecord.setSimpleField("key2", "value2");
        // records.add(newRecord);
        znrecordUpdaters.add(new ZNRecordUpdater(newRecord));
    }
    success = accessor.updateChildren(paths, znrecordUpdaters, AccessOption.PERSISTENT);
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        Assert.assertTrue(success[i], "Should succeed in update " + msgId);
    }
    // test get what we updated
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        String path = PropertyPathBuilder.instanceMessage(root, "host_1", msgId);
        ZNRecord record = _gZkClient.readData(path);
        Assert.assertEquals(record.getSimpleFields().size(), 2, "Should have 2 simple fields set");
        Assert.assertEquals(record.getSimpleField("key2"), "value2", "Should have value2 set");
    }
    // test async getChildren
    String parentPath = PropertyPathBuilder.instanceMessage(root, "host_1");
    records = accessor.getChildren(parentPath, null, 0, 0, 0);
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        ZNRecord record = records.get(i);
        Assert.assertEquals(record.getId(), msgId, "Should get what we updated");
        Assert.assertEquals(record.getSimpleFields().size(), 2, "Should have 2 simple fields set");
        Assert.assertEquals(record.getSimpleField("key2"), "value2", "Should have value2 set");
    }
    // test async exists
    paths = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        paths.add(PropertyPathBuilder.instanceMessage(root, "host_1", msgId));
    }
    boolean[] exists = accessor.exists(paths, 0);
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        Assert.assertTrue(exists[i], "Should exist " + msgId);
    }
    // test async getStats
    paths = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        paths.add(PropertyPathBuilder.instanceMessage(root, "host_1", msgId));
    }
    Stat[] stats = accessor.getStats(paths, 0);
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        Assert.assertNotNull(stats[i], "Stat should exist for " + msgId);
        Assert.assertEquals(stats[i].getVersion(), 2, "DataVersion should be 2, since we set 1 and update 1 for " + msgId);
    }
    // test async remove
    paths = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        paths.add(PropertyPathBuilder.instanceMessage(root, "host_1", msgId));
    }
    success = accessor.remove(paths, 0);
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        Assert.assertTrue(success[i], "Should succeed in remove " + msgId);
    }
    // test get what we removed
    for (int i = 0; i < 10; i++) {
        String msgId = "msg_" + i;
        String path = PropertyPathBuilder.instanceMessage(root, "host_1", msgId);
        boolean pathExists = _gZkClient.exists(path);
        Assert.assertFalse(pathExists, "Should be removed " + msgId);
    }
    System.out.println("END TestZkBaseDataAccessor.async at " + new Date(System.currentTimeMillis()));
}

Example 14 with DataUpdater

use of org.apache.helix.zookeeper.zkclient.DataUpdater in project helix by apache.

the class TestWtCacheAsyncOpSingleThread method testHappyPathZkCacheBaseDataAccessor.

@Test
public void testHappyPathZkCacheBaseDataAccessor() {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    // init zkCacheDataAccessor
    String curStatePath = PropertyPathBuilder.instanceCurrentState(clusterName, "localhost_8901");
    String extViewPath = PropertyPathBuilder.externalView(clusterName);
    ZkBaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<>(_gZkClient);
    baseAccessor.create(curStatePath, null, AccessOption.PERSISTENT);
    List<String> cachePaths = Arrays.asList(curStatePath, extViewPath);
    ZkCacheBaseDataAccessor<ZNRecord> accessor = new ZkCacheBaseDataAccessor<>(baseAccessor, null, cachePaths, null);
    boolean ret = TestHelper.verifyZkCache(cachePaths, accessor._wtCache._cache, _gZkClient, false);
    Assert.assertTrue(ret, "wtCache doesn't match data on Zk");
    // create 10 current states
    List<String> paths = new ArrayList<>();
    List<ZNRecord> records = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String path = PropertyPathBuilder.instanceCurrentState(clusterName, "localhost_8901", "session_0", "TestDB" + i);
        ZNRecord record = new ZNRecord("TestDB" + i);
        paths.add(path);
        records.add(record);
    }
    boolean[] success = accessor.createChildren(paths, records, AccessOption.PERSISTENT);
    for (int i = 0; i < 10; i++) {
        Assert.assertTrue(success[i], "Should succeed in create: " + paths.get(i));
    }
    // verify wtCache
    // TestHelper.printCache(accessor._wtCache);
    ret = TestHelper.verifyZkCache(cachePaths, accessor._wtCache._cache, _gZkClient, false);
    Assert.assertTrue(ret, "wtCache doesn't match data on Zk");
    // update each current state 10 times
    List<DataUpdater<ZNRecord>> updaters = new ArrayList<>();
    for (int j = 0; j < 10; j++) {
        paths.clear();
        updaters.clear();
        for (int i = 0; i < 10; i++) {
            String path = curStatePath + "/session_0/TestDB" + i;
            ZNRecord newRecord = new ZNRecord("TestDB" + i);
            newRecord.setSimpleField("" + j, "" + j);
            DataUpdater<ZNRecord> updater = new ZNRecordUpdater(newRecord);
            paths.add(path);
            updaters.add(updater);
        }
        success = accessor.updateChildren(paths, updaters, AccessOption.PERSISTENT);
        for (int i = 0; i < 10; i++) {
            Assert.assertTrue(success[i], "Should succeed in update: " + paths.get(i));
        }
    }
    // verify cache
    // TestHelper.printCache(accessor._wtCache);
    ret = TestHelper.verifyZkCache(cachePaths, accessor._wtCache._cache, _gZkClient, false);
    Assert.assertTrue(ret, "wtCache doesn't match data on Zk");
    // set 10 external views
    paths.clear();
    records.clear();
    for (int i = 0; i < 10; i++) {
        String path = PropertyPathBuilder.externalView(clusterName, "TestDB" + i);
        ZNRecord record = new ZNRecord("TestDB" + i);
        paths.add(path);
        records.add(record);
    }
    success = accessor.setChildren(paths, records, AccessOption.PERSISTENT);
    for (int i = 0; i < 10; i++) {
        Assert.assertTrue(success[i], "Should succeed in set: " + paths.get(i));
    }
    // verify wtCache
    // TestHelper.printCache(accessor._wtCache);
    ret = TestHelper.verifyZkCache(cachePaths, accessor._wtCache._cache, _gZkClient, false);
    Assert.assertTrue(ret, "wtCache doesn't match data on Zk");
    // get 10 external views
    paths.clear();
    records.clear();
    for (int i = 0; i < 10; i++) {
        String path = PropertyPathBuilder.externalView(clusterName, "TestDB" + i);
        paths.add(path);
    }
    records = accessor.get(paths, null, 0, true);
    for (int i = 0; i < 10; i++) {
        Assert.assertEquals(records.get(i).getId(), "TestDB" + i);
    }
    // getChildren
    records.clear();
    records = accessor.getChildren(extViewPath, null, 0, 0, 0);
    for (int i = 0; i < 10; i++) {
        Assert.assertEquals(records.get(i).getId(), "TestDB" + i);
    }
    // exists
    paths.clear();
    for (int i = 0; i < 10; i++) {
        String path = PropertyPathBuilder.instanceCurrentState(clusterName, "localhost_8901", "session_0", "TestDB" + i);
        paths.add(path);
    }
    success = accessor.exists(paths, 0);
    for (int i = 0; i < 10; i++) {
        Assert.assertTrue(success[i], "Should exits: TestDB" + i);
    }
    deleteCluster(clusterName);
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 15 with DataUpdater

use of org.apache.helix.zookeeper.zkclient.DataUpdater in project helix by apache.

the class TaskDriver method setSingleWorkflowTargetState.

/**
 * Helper function to change target state for a given workflow
 */
private void setSingleWorkflowTargetState(String workflow, final TargetState state) {
    LOG.info("Set " + workflow + " to target state " + state);
    WorkflowConfig workflowConfig = TaskUtil.getWorkflowConfig(_accessor, workflow);
    if (workflowConfig == null) {
        LOG.warn("WorkflowConfig for {} not found!", workflow);
        return;
    }
    WorkflowContext workflowContext = TaskUtil.getWorkflowContext(_propertyStore, workflow);
    if (state != TargetState.DELETE && workflowContext != null && workflowContext.getFinishTime() != WorkflowContext.UNFINISHED) {
        // Should not update target state for completed workflow
        LOG.info("Workflow {} is already completed, skip to update its target state {}", workflow, state);
        return;
    }
    DataUpdater<ZNRecord> updater = currentData -> {
        if (currentData != null) {
            currentData.setSimpleField(WorkflowConfig.WorkflowConfigProperty.TargetState.name(), state.name());
        } else {
            LOG.warn("TargetState DataUpdater: Fails to update target state for {}. CurrentData is null.", workflow);
        }
        return currentData;
    };
    PropertyKey workflowConfigKey = TaskUtil.getWorkflowConfigKey(_accessor, workflow);
    _accessor.getBaseDataAccessor().update(workflowConfigKey.getPath(), updater, AccessOption.PERSISTENT);
}