Examples with Set java.util.Set used on opensource projects

Example 51 with Set

use of java.util.Set in project hadoop by apache.

the class CommonNodeLabelsManager method normalizeNodeIdToLabels.

protected Map<NodeId, Set<String>> normalizeNodeIdToLabels(Map<NodeId, Set<String>> nodeIdToLabels) {
    Map<NodeId, Set<String>> newMap = new TreeMap<NodeId, Set<String>>();
    for (Entry<NodeId, Set<String>> entry : nodeIdToLabels.entrySet()) {
        NodeId id = entry.getKey();
        Set<String> labels = entry.getValue();
        newMap.put(id, normalizeLabels(labels));
    }
    return newMap;
}

Example 52 with Set

use of java.util.Set in project hadoop by apache.

the class FileSystemNodeLabelsStore method loadFromMirror.

protected void loadFromMirror(Path newMirrorPath, Path oldMirrorPath) throws IOException {
    // If mirror.new exists, read from mirror.new,
    FSDataInputStream is = null;
    try {
        is = fs.open(newMirrorPath);
    } catch (FileNotFoundException e) {
        try {
            is = fs.open(oldMirrorPath);
        } catch (FileNotFoundException ignored) {
        }
    }
    if (null != is) {
        List<NodeLabel> labels = new AddToClusterNodeLabelsRequestPBImpl(AddToClusterNodeLabelsRequestProto.parseDelimitedFrom(is)).getNodeLabels();
        mgr.addToCluserNodeLabels(labels);
        if (mgr.isCentralizedConfiguration()) {
            // Only load node to labels mapping while using centralized configuration
            Map<NodeId, Set<String>> nodeToLabels = new ReplaceLabelsOnNodeRequestPBImpl(ReplaceLabelsOnNodeRequestProto.parseDelimitedFrom(is)).getNodeToLabels();
            mgr.replaceLabelsOnNode(nodeToLabels);
        }
        is.close();
    }
}

Example 53 with Set

use of java.util.Set in project hadoop by apache.

the class NonAppendableFSNodeLabelStore method writeNewMirror.

private void writeNewMirror() throws IOException {
    ReentrantReadWriteLock.ReadLock readLock = mgr.readLock;
    try {
        // Acquire readlock to make sure we get cluster node labels and
        // node-to-labels mapping atomically.
        readLock.lock();
        List<NodeLabel> nodeLabels = mgr.getClusterNodeLabels();
        Map<NodeId, Set<String>> nodeToLabels = mgr.getNodeLabels();
        // Write mirror to mirror.new.tmp file
        Path newTmpPath = new Path(fsWorkingPath, MIRROR_FILENAME + ".new.tmp");
        FSDataOutputStream os = fs.create(newTmpPath, true);
        ((AddToClusterNodeLabelsRequestPBImpl) AddToClusterNodeLabelsRequest.newInstance(nodeLabels)).getProto().writeDelimitedTo(os);
        if (mgr.isCentralizedConfiguration()) {
            // Only save node-to-labels mapping while using centralized configuration
            ((ReplaceLabelsOnNodeRequestPBImpl) ReplaceLabelsOnNodeRequest.newInstance(nodeToLabels)).getProto().writeDelimitedTo(os);
        }
        os.close();
        // Rename mirror.new.tmp to mirror.new (will remove .new if it's existed)
        Path newPath = new Path(fsWorkingPath, MIRROR_FILENAME + ".new");
        fs.delete(newPath, false);
        fs.rename(newTmpPath, newPath);
        // Remove existing mirror and rename mirror.new to mirror
        Path mirrorPath = new Path(fsWorkingPath, MIRROR_FILENAME);
        fs.delete(mirrorPath, false);
        fs.rename(newPath, mirrorPath);
    } finally {
        readLock.unlock();
    }
}

Example 54 with Set

use of java.util.Set in project hadoop by apache.

the class TestRMRestart method testRMRestartRecoveringNodeLabelManager.

// Test does following verification
// 1. Start RM1 with store patch /tmp
// 2. Add/remove/replace labels to cluster and node lable and verify
// 3. Start RM2 with store patch /tmp only
// 4. Get cluster and node lobel, it should be present by recovering it
@Test(timeout = 20000)
public void testRMRestartRecoveringNodeLabelManager() throws Exception {
    // Initial FS node label store root dir to a random tmp dir
    File nodeLabelFsStoreDir = new File("target", this.getClass().getSimpleName() + "-testRMRestartRecoveringNodeLabelManager");
    if (nodeLabelFsStoreDir.exists()) {
        FileUtils.deleteDirectory(nodeLabelFsStoreDir);
    }
    nodeLabelFsStoreDir.deleteOnExit();
    String nodeLabelFsStoreDirURI = nodeLabelFsStoreDir.toURI().toString();
    conf.set(YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR, nodeLabelFsStoreDirURI);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
    MockRM rm1 = new MockRM(conf, memStore) {

        @Override
        protected RMNodeLabelsManager createNodeLabelManager() {
            RMNodeLabelsManager mgr = new RMNodeLabelsManager();
            mgr.init(getConfig());
            return mgr;
        }
    };
    rm1.init(conf);
    rm1.start();
    RMNodeLabelsManager nodeLabelManager = rm1.getRMContext().getNodeLabelManager();
    Set<String> clusterNodeLabels = new HashSet<String>();
    clusterNodeLabels.add("x");
    clusterNodeLabels.add("y");
    clusterNodeLabels.add("z");
    // Add node label x,y,z
    nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(clusterNodeLabels);
    // Add node Label to Node h1->x
    NodeId n1 = NodeId.newInstance("h1", 0);
    nodeLabelManager.addLabelsToNode(ImmutableMap.of(n1, toSet("x")));
    clusterNodeLabels.remove("z");
    // Remove cluster label z
    nodeLabelManager.removeFromClusterNodeLabels(toSet("z"));
    // Replace nodelabel h1->x,y
    nodeLabelManager.replaceLabelsOnNode(ImmutableMap.of(n1, toSet("y")));
    // Wait for updating store.It is expected NodeStore update should happen
    // very fast since it has separate dispatcher. So waiting for max 5 seconds,
    // which is sufficient time to update NodeStore.
    int count = 10;
    while (count-- > 0) {
        if (nodeLabelManager.getNodeLabels().size() > 0) {
            break;
        }
        Thread.sleep(500);
    }
    Assert.assertEquals(clusterNodeLabels.size(), nodeLabelManager.getClusterNodeLabelNames().size());
    Map<NodeId, Set<String>> nodeLabels = nodeLabelManager.getNodeLabels();
    Assert.assertEquals(1, nodeLabelManager.getNodeLabels().size());
    Assert.assertTrue(nodeLabels.get(n1).equals(toSet("y")));
    MockRM rm2 = new MockRM(conf, memStore) {

        @Override
        protected RMNodeLabelsManager createNodeLabelManager() {
            RMNodeLabelsManager mgr = new RMNodeLabelsManager();
            mgr.init(getConfig());
            return mgr;
        }
    };
    rm2.init(conf);
    rm2.start();
    nodeLabelManager = rm2.getRMContext().getNodeLabelManager();
    Assert.assertEquals(clusterNodeLabels.size(), nodeLabelManager.getClusterNodeLabelNames().size());
    nodeLabels = nodeLabelManager.getNodeLabels();
    Assert.assertEquals(1, nodeLabelManager.getNodeLabels().size());
    Assert.assertTrue(nodeLabels.get(n1).equals(toSet("y")));
    rm1.stop();
    rm2.stop();
}

Example 55 with Set

use of java.util.Set in project hadoop by apache.

the class TestRMAdminService method testRemoveClusterNodeLabelsWithCentralizedConfigurationDisabled.

@Test
public void testRemoveClusterNodeLabelsWithCentralizedConfigurationDisabled() throws IOException, YarnException {
    // create RM and set it's ACTIVE
    MockRM rm = new MockRM();
    ((RMContextImpl) rm.getRMContext()).setHAServiceState(HAServiceState.ACTIVE);
    RMNodeLabelsManager labelMgr = rm.rmContext.getNodeLabelManager();
    rm.adminService.isCentralizedNodeLabelConfiguration = false;
    // by default, distributed configuration for node label is disabled, this
    // should pass
    labelMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y"));
    rm.adminService.removeFromClusterNodeLabels(RemoveFromClusterNodeLabelsRequest.newInstance((Set<String>) ImmutableSet.of("x")));
    Set<String> clusterNodeLabels = labelMgr.getClusterNodeLabelNames();
    assertEquals(1, clusterNodeLabels.size());
    rm.close();
}