Examples with ClusterId net.dempsy.config.ClusterId used on opensource projects

Example 16 with ClusterId

use of net.dempsy.config.ClusterId in project Dempsy by Dempsy.

the class TestElasticity method testExpansionPassivation.

@Test
public void testExpansionPassivation() throws Exception {
    final String[][] actxPath = { { "elasticity/adaptor.xml" }, { "elasticity/mp-num-count.xml" }, { "elasticity/mp-num-rank.xml" } };
    runCombos("testExpansionPassivation", (r, c, s, t, ser) -> isElasticRoutingStrategy(r), actxPath, ns -> {
        final AtomicBoolean keepGoing = new AtomicBoolean(true);
        try {
            final List<NodeManagerWithContext> nodes = new ArrayList<>(ns.nodes);
            // grab the adaptor from the 0'th cluster + the 0'th (only) node.
            final NumberProducer adaptor = nodes.get(0).ctx.getBean(NumberProducer.class);
            // grab access to the Dispatcher from the Adaptor
            final Dispatcher dispatcher = adaptor.dispatcher;
            final AtomicInteger rankIndexToSend = new AtomicInteger(0);
            final Runnable sendMessages = () -> {
                // for an integer is the integer itself so we can get every shard by sending
                try {
                    while (keepGoing.get()) {
                        for (int num = 0; num < 20; num++) {
                            final int number = num;
                            dispatcher.dispatch(uncheck(() -> ke.extract(new Number(number, rankIndexToSend.get()))));
                        }
                    }
                } catch (final InterruptedException ie) {
                    if (keepGoing.get())
                        LOGGER.info("send message thread interrupted but still running.");
                }
            };
            try (final ClusterInfoSession session = ns.sessionFactory.createSession()) {
                waitForEvenShardDistribution(session, "test-cluster1", 1, nodes);
                final NumberRank rank = nodes.get(2).ctx.getBean(NumberRank.class);
                runACycle(keepGoing, rankIndexToSend.get(), rank, sendMessages);
                // now we have 20 Mps in test-cluster1
                final ClusterMetricGetters sc = (ClusterMetricGetters) nodes.get(1).manager.getClusterStatsCollector(new ClusterId(currentAppName, "test-cluster1"));
                assertEquals(20L, sc.getMessageProcessorCount());
                // add a second node for the cluster test-cluster1
                nodes.add(makeNode(new String[] { "elasticity/mp-num-count.xml" }));
                waitForEvenShardDistribution(session, "test-cluster1", 2, nodes);
                // about 1/2 should drop out.
                assertTrue(poll(o -> {
                    // System.err.println(sc.getMessageProcessorCount());
                    return sc.getMessageProcessorCount() < 15L;
                }));
            }
        } finally {
            keepGoing.set(false);
        }
    });
}

Example 17 with ClusterId

use of net.dempsy.config.ClusterId in project Dempsy by Dempsy.

the class TestResourceManagement method executeSimpleWordCountTest.

private void executeSimpleWordCountTest(final String[][] ctxs, final String testName) throws Throwable {
    try (@SuppressWarnings("resource") final SystemPropertyManager props = new SystemPropertyManager().set("min_nodes", "1")) {
        // need to make it wait.
        WordProducer.latch = new CountDownLatch(1);
        runCombos(testName, ctxs, new TestToRun() {

            @Override
            public void test(final Nodes n) throws Exception {
                clear();
                try {
                    final List<NodeManagerWithContext> nodes = n.nodes;
                    final NodeManager manager1 = nodes.get(0).manager;
                    // wait until I can reach the cluster from the adaptor.
                    assertTrue(poll(o -> manager1.getRouter().allReachable("test-cluster1").size() == 1));
                    final ClassPathXmlApplicationContext ctx = nodes.get(0).ctx;
                    final WordProducer adaptor;
                    final ClusterMetricGetters stats;
                    WordProducer.latch.countDown();
                    adaptor = ctx.getBean(WordProducer.class);
                    // somtimes there's 1 node, sometimes there's 2
                    final NodeManager manager2 = nodes.size() > 1 ? nodes.get(1).manager : manager1;
                    stats = (ClusterMetricGetters) manager2.getClusterStatsCollector(new ClusterId(currentAppName, "test-cluster1"));
                    assertTrue(poll(o -> adaptor.done.get()));
                    assertTrue(poll(o -> {
                        // System.out.println("" + adaptor.numDispatched + " == " + stats.getProcessedMessageCount());
                        return adaptor.numDispatched == stats.getProcessedMessageCount() + stats.getMessageDiscardedCount();
                    }));
                    // check that all of the Word instances have been freed
                    Word.allWords.forEach(w -> assertFalse(w.amIOpen));
                    Word.allWords.forEach(w -> assertEquals(0, w.refCount()));
                    WordCount.allWordCounts.forEach(w -> assertFalse(w.amIOpen));
                    WordCount.allWordCounts.forEach(w -> assertEquals(0, w.refCount()));
                } finally {
                    clear();
                }
            }

            @Override
            public void postShutdown() throws InterruptedException {
                clear();
            }
        });
    }
}

Example 18 with ClusterId

use of net.dempsy.config.ClusterId in project Dempsy by Dempsy.

the class TestWordCount method testWordCountNoRankAdaptorOnlyNode.

@Test
public void testWordCountNoRankAdaptorOnlyNode() throws Throwable {
    try (@SuppressWarnings("resource") final SystemPropertyManager props = new SystemPropertyManager().set("min_nodes", "2")) {
        final String[][] ctxs = { // adaptor only node
        { "classpath:/word-count/adaptor-kjv.xml" }, { "classpath:/word-count/mp-word-count.xml" }, { "classpath:/word-count/mp-word-count.xml" }, { "classpath:/word-count/mp-word-count.xml" }, { "classpath:/word-count/mp-word-count.xml" }, { "classpath:/word-count/mp-word-count.xml" } };
        // the adaptor is the first one.
        final int NUM_WC = ctxs.length - 1;
        WordCounter.aliveWordsToTestPassivate.clear();
        // need to make it wait.
        WordProducer.latch = new CountDownLatch(1);
        runCombos("testWordCountNoRankMultinode", (r, c, s, t, ser) -> isElasticRoutingStrategy(r) && isContainerOkay(c), ctxs, n -> {
            final ClusterId countCId = new ClusterId(currentAppName, "test-cluster1");
            final List<NodeManagerWithContext> nodes = n.nodes;
            final NodeManager[] managers = nodes.stream().map(nm -> nm.manager).toArray(NodeManager[]::new);
            // wait until I can reach the cluster from the adaptor.
            assertTrue(poll(o -> managers[0].getRouter().allReachable("test-cluster1").size() == NUM_WC));
            WordProducer.latch.countDown();
            final WordProducer adaptor = nodes.get(0).ctx.getBean(WordProducer.class);
            waitForAllSent(adaptor);
            assertTrue(poll(o -> adaptor.done.get()));
            final var statsByNode = Arrays.asList(managers).subList(1, managers.length).stream().map(nm -> Pair.with((NodeMetricGetters) nm.getNodeStatsCollector(), (ClusterMetricGetters) nm.getClusterStatsCollector(countCId))).collect(Collectors.toList());
            final NodeMetricGetters adaptorStats = (NodeMetricGetters) managers[0].getNodeStatsCollector();
            assertTrue(poll(o -> {
                final int allDiscardedMessagesCount = allDiscardedMessagesCount(adaptorStats, castToTupleList2(statsByNode));
                return adaptor.numDispatched == allDiscardedMessagesCount + statsByNode.stream().mapToInt(sc -> (int) sc.getValue1().getDispatchedMessageCount()).sum();
            }));
        });
        final int remaining = WordCounter.aliveWordsToTestPassivate.size();
        WordCounter.aliveWordsToTestPassivate.clear();
        assertEquals(0, remaining);
    }
}

Example 19 with ClusterId

use of net.dempsy.config.ClusterId in project Dempsy by Dempsy.

the class TestWordCount method testWordCountWithRank.

@Test
public void testWordCountWithRank() throws Throwable {
    try (@SuppressWarnings("resource") final SystemPropertyManager props = new SystemPropertyManager().set("min_nodes", "1")) {
        final String[][] ctxs = { { "classpath:/word-count/adaptor-kjv.xml", "classpath:/word-count/mp-word-count.xml", "classpath:/word-count/mp-word-rank.xml" } };
        WordCounter.aliveWordsToTestPassivate.clear();
        // need to make it wait.
        WordProducer.latch = new CountDownLatch(1);
        runCombos("testWordCountWithRank", (r, c, s, t, ser) -> isContainerOkay(c), ctxs, n -> {
            final ClusterId countCId = new ClusterId(currentAppName, "test-cluster1");
            final ClusterId rankCId = new ClusterId(currentAppName, "test-cluster2");
            final List<NodeManagerWithContext> nodes = n.nodes;
            final NodeManager manager = nodes.get(0).manager;
            // wait until I can reach the cluster from the adaptor.
            assertTrue(poll(o -> manager.getRouter().allReachable("test-cluster1").size() == 1));
            final ClassPathXmlApplicationContext ctx = nodes.get(0).ctx;
            final WordProducer adaptor;
            WordProducer.latch.countDown();
            adaptor = ctx.getBean(WordProducer.class);
            assertTrue(poll(o -> adaptor.done.get()));
            final var statsByNode = Triplet.with((NodeMetricGetters) manager.getNodeStatsCollector(), (ClusterMetricGetters) manager.getClusterStatsCollector(countCId), (ClusterMetricGetters) manager.getClusterStatsCollector(rankCId));
            // wait until all of the counts are also passed to WordRank
            assertTrue(poll(o -> {
                final int allDiscardedMessagesCount = allDiscardedMessagesCount(null, castToTupleList3(List.of(statsByNode)));
                final ClusterMetricGetters rstat = statsByNode.getValue2();
                return adaptor.numDispatched == allDiscardedMessagesCount + rstat.getDispatchedMessageCount();
            }));
            stopSystem();
            // pull the Rank mp from the manager
            final MessageProcessorLifecycle<?> mp = AccessUtil.getMp(manager, "test-cluster2");
            @SuppressWarnings("unchecked") final WordRank prototype = ((MessageProcessor<WordRank>) mp).getPrototype();
            final List<Rank> ranks = prototype.getPairs();
            Collections.sort(ranks, (o1, o2) -> o2.rank.compareTo(o1.rank));
            final int top10OrSo = (ranks.size() < 10) ? ranks.size() : 10;
            final List<Rank> top10 = ranks.subList(0, top10OrSo);
            final String errStr = "expected the top 10 results: " + top10 + " to all be in:" + finalResults;
            top10.stream().forEach(r -> assertTrue(errStr, finalResults.contains(r.word)));
        });
        final int remaining = WordCounter.aliveWordsToTestPassivate.size();
        WordCounter.aliveWordsToTestPassivate.clear();
        assertEquals(0, remaining);
    }
}

Example 20 with ClusterId

use of net.dempsy.config.ClusterId in project Dempsy by Dempsy.

the class TestContainer method testConfiguration.

@Test
public void testConfiguration() throws Exception {
    // this assertion is superfluous, since we deref container in setUp()
    assertNotNull("did not create container", container);
    assertEquals(new ClusterId("test-app", "test-cluster"), container.getClusterId());
    final TestProcessor prototype = context.getBean(TestProcessor.class);
    assertEquals(1, prototype.startCalled.get());
    assertNotNull(prototype.clusterId);
}