Examples with Adaptor net.dempsy.messages.Adaptor used on opensource projects

Example 11 with Adaptor

use of net.dempsy.messages.Adaptor in project Dempsy by Dempsy.

the class TestConfig method testConfigAdaptorWithKeyStoreBuilder.

@Test(expected = IllegalStateException.class)
public void testConfigAdaptorWithKeyStoreBuilder() throws Throwable {
    final Node node = new Node.Builder("test").defaultRoutingStrategyId("").cluster("test-slot").adaptor(new Adaptor() {

        @Override
        public void stop() {
        }

        @Override
        public void start() {
        }

        @Override
        public void setDispatcher(final Dispatcher dispatcher) {
        }
    }).keySource(new KeySource<Object>() {

        @Override
        public Iterable<Object> getAllPossibleKeys() {
            return null;
        }
    }).build();
    node.validate();
}

Example 12 with Adaptor

use of net.dempsy.messages.Adaptor in project Dempsy by Dempsy.

the class TestElasticity method testForProfiler.

@Test
public void testForProfiler() throws Throwable {
    try {
        // set up the test.
        final Number[] numbers = new Number[profilerTestNumberCount];
        final Random random = new Random();
        for (int i = 0; i < numbers.length; i++) numbers[i] = new Number(random.nextInt(1000), 0);
        final KeyExtractor ke = new KeyExtractor();
        runCombos("testForProfiler", (r, c, s, t, ser) -> isElasticRoutingStrategy(r), actxPath, new String[][][] { null, { { "min_nodes", "3" } }, { { "min_nodes", "3" } }, { { "min_nodes", "3" } }, null }, ns -> {
            final List<NodeManagerWithContext> nodes = ns.nodes;
            // Grab the one NumberRank Mp from the single Node in the third (0 base 2nd) cluster.
            final NumberRank rank = nodes.get(4).ctx.getBean(NumberRank.class);
            try (final ClusterInfoSession session = ns.sessionFactory.createSession()) {
                waitForEvenShardDistribution(session, "test-cluster1", 3, nodes);
                // get the Adaptor Router's statCollector
                final List<NodeMetricGetters> scs = nodes.stream().map(nwm -> nwm.manager).map(nm -> nm.getNodeStatsCollector()).map(sc -> (NodeMetricGetters) sc).collect(Collectors.toList());
                // grab the adaptor from the 0'th cluster + the 0'th (only) node.
                final NumberProducer adaptor = nodes.get(0).ctx.getBean(NumberProducer.class);
                // grab access to the Dispatcher from the Adaptor
                final Dispatcher dispatcher = adaptor.dispatcher;
                final long startTime = System.currentTimeMillis();
                for (int i = 0; i < numbers.length; i++) dispatcher.dispatch(ke.extract(numbers[i]));
                LOGGER.info("====> Checking exact count.");
                // keep going as long as they are trickling in.
                assertTrue(() -> {
                    IntStream.range(0, scs.size()).forEach(i -> {
                        System.out.println("======> " + i);
                        final NodeMetricGetters mg = scs.get(i);
                        if (mg != null) {
                            System.out.println("discarded: " + mg.getDiscardedMessageCount());
                            System.out.println("not sent: " + mg.getMessagesNotSentCount());
                        }
                    });
                    return "expected: " + profilerTestNumberCount + " but was: " + (rank.totalMessages.get() + scs.get(0).getMessagesNotSentCount()) + ", (delivered: " + rank.totalMessages.get() + ", not sent: " + scs.get(0).getMessagesNotSentCount() + ")";
                }, poll(o -> profilerTestNumberCount == (rank.totalMessages.get() + scs.stream().map(sc -> Long.valueOf(sc.getMessagesNotSentCount())).reduce(Long.valueOf(0), (v1, v2) -> Long.valueOf(v1.longValue() + v2.longValue()).longValue()))));
                // assert that at least SOMETHING went through
                assertTrue(rank.totalMessages.get() > 0);
                LOGGER.info("testForProfiler time " + (System.currentTimeMillis() - startTime));
                @SuppressWarnings("unchecked") final AtomicLong count = nodes.stream().map(// get the NumberCounter Mp
                nmwc -> (MessageProcessor<NumberCounter>) nmwc.manager.getMp("test-cluster1")).filter(// if it exists
                l -> l != null).map(// pull the prototype
                l -> l.getPrototype().messageCount).reduce(new AtomicLong(0), // sum up all of the counts
                (v1, v2) -> new AtomicLong(v1.get() + v2.get()));
                assertEquals(profilerTestNumberCount, count.get());
            }
        });
    } catch (final Throwable th) {
        th.printStackTrace();
        throw th;
    }
}

Example 13 with Adaptor

use of net.dempsy.messages.Adaptor in project Dempsy by Dempsy.

the class TestElasticity method testExpansionPassivation.

@Test
public void testExpansionPassivation() throws Exception {
    final String[][] actxPath = { { "elasticity/adaptor.xml" }, { "elasticity/mp-num-count.xml" }, { "elasticity/mp-num-rank.xml" } };
    runCombos("testExpansionPassivation", (r, c, s, t, ser) -> isElasticRoutingStrategy(r), actxPath, ns -> {
        final AtomicBoolean keepGoing = new AtomicBoolean(true);
        try {
            final List<NodeManagerWithContext> nodes = new ArrayList<>(ns.nodes);
            // grab the adaptor from the 0'th cluster + the 0'th (only) node.
            final NumberProducer adaptor = nodes.get(0).ctx.getBean(NumberProducer.class);
            // grab access to the Dispatcher from the Adaptor
            final Dispatcher dispatcher = adaptor.dispatcher;
            final AtomicInteger rankIndexToSend = new AtomicInteger(0);
            final Runnable sendMessages = () -> {
                // for an integer is the integer itself so we can get every shard by sending
                try {
                    while (keepGoing.get()) {
                        for (int num = 0; num < 20; num++) {
                            final int number = num;
                            dispatcher.dispatch(uncheck(() -> ke.extract(new Number(number, rankIndexToSend.get()))));
                        }
                    }
                } catch (final InterruptedException ie) {
                    if (keepGoing.get())
                        LOGGER.info("send message thread interrupted but still running.");
                }
            };
            try (final ClusterInfoSession session = ns.sessionFactory.createSession()) {
                waitForEvenShardDistribution(session, "test-cluster1", 1, nodes);
                final NumberRank rank = nodes.get(2).ctx.getBean(NumberRank.class);
                runACycle(keepGoing, rankIndexToSend.get(), rank, sendMessages);
                // now we have 20 Mps in test-cluster1
                final ClusterMetricGetters sc = (ClusterMetricGetters) nodes.get(1).manager.getClusterStatsCollector(new ClusterId(currentAppName, "test-cluster1"));
                assertEquals(20L, sc.getMessageProcessorCount());
                // add a second node for the cluster test-cluster1
                nodes.add(makeNode(new String[] { "elasticity/mp-num-count.xml" }));
                waitForEvenShardDistribution(session, "test-cluster1", 2, nodes);
                // about 1/2 should drop out.
                assertTrue(poll(o -> {
                    // System.err.println(sc.getMessageProcessorCount());
                    return sc.getMessageProcessorCount() < 15L;
                }));
            }
        } finally {
            keepGoing.set(false);
        }
    });
}

Example 14 with Adaptor

use of net.dempsy.messages.Adaptor in project Dempsy by Dempsy.

the class TestResourceManagement method executeSimpleWordCountTest.

private void executeSimpleWordCountTest(final String[][] ctxs, final String testName) throws Throwable {
    try (@SuppressWarnings("resource") final SystemPropertyManager props = new SystemPropertyManager().set("min_nodes", "1")) {
        // need to make it wait.
        WordProducer.latch = new CountDownLatch(1);
        runCombos(testName, ctxs, new TestToRun() {

            @Override
            public void test(final Nodes n) throws Exception {
                clear();
                try {
                    final List<NodeManagerWithContext> nodes = n.nodes;
                    final NodeManager manager1 = nodes.get(0).manager;
                    // wait until I can reach the cluster from the adaptor.
                    assertTrue(poll(o -> manager1.getRouter().allReachable("test-cluster1").size() == 1));
                    final ClassPathXmlApplicationContext ctx = nodes.get(0).ctx;
                    final WordProducer adaptor;
                    final ClusterMetricGetters stats;
                    WordProducer.latch.countDown();
                    adaptor = ctx.getBean(WordProducer.class);
                    // somtimes there's 1 node, sometimes there's 2
                    final NodeManager manager2 = nodes.size() > 1 ? nodes.get(1).manager : manager1;
                    stats = (ClusterMetricGetters) manager2.getClusterStatsCollector(new ClusterId(currentAppName, "test-cluster1"));
                    assertTrue(poll(o -> adaptor.done.get()));
                    assertTrue(poll(o -> {
                        // System.out.println("" + adaptor.numDispatched + " == " + stats.getProcessedMessageCount());
                        return adaptor.numDispatched == stats.getProcessedMessageCount() + stats.getMessageDiscardedCount();
                    }));
                    // check that all of the Word instances have been freed
                    Word.allWords.forEach(w -> assertFalse(w.amIOpen));
                    Word.allWords.forEach(w -> assertEquals(0, w.refCount()));
                    WordCount.allWordCounts.forEach(w -> assertFalse(w.amIOpen));
                    WordCount.allWordCounts.forEach(w -> assertEquals(0, w.refCount()));
                } finally {
                    clear();
                }
            }

            @Override
            public void postShutdown() throws InterruptedException {
                clear();
            }
        });
    }
}

Example 15 with Adaptor

use of net.dempsy.messages.Adaptor in project Dempsy by Dempsy.

the class TestWordCount method runTestWithOutputCycle.

private void runTestWithOutputCycle(final String testName, final String outputSchedCtx) throws Exception {
    final String[][] ctxs = { // adaptor only node
    { "classpath:/word-count/adaptor-kjv.xml" }, { "classpath:/word-count/mp-word-count.xml", "classpath:/word-count/mp-word-count-output.xml", outputSchedCtx }, { "classpath:/word-count/mp-word-count.xml", "classpath:/word-count/mp-word-count-output.xml", outputSchedCtx }, { "classpath:/word-count/mp-rank-catcher.xml" } };
    // the adaptor is the first one, rank catcher the last.
    final int NUM_WC = ctxs.length - 2;
    try (@SuppressWarnings("resource") final SystemPropertyManager props = new SystemPropertyManager().set("min_nodes", Integer.toString(NUM_WC)).set("routing-group", ":group").set("send_threads", "1").set("receive_threads", "1").set("blocking-queue-size", "500000")) {
        // need to make it wait.
        WordProducer.latch = new CountDownLatch(1);
        runCombos(testName, (r, c, s, t, ser) -> isElasticRoutingStrategy(r) && isContainerOkay(c), ctxs, n -> {
            final List<NodeManagerWithContext> nodes = n.nodes;
            final NodeManager[] managers = nodes.stream().map(nm -> nm.manager).toArray(NodeManager[]::new);
            // wait until I can reach the cluster from the adaptor.
            assertTrue(poll(o -> managers[0].getRouter().allReachable("test-cluster1").size() == NUM_WC));
            assertTrue(poll(o -> managers[0].getRouter().allReachable("test-cluster2").size() == NUM_WC));
            assertTrue(poll(o -> managers[0].getRouter().allReachable("test-cluster3").size() == 1));
            for (int i = 0; i < NUM_WC; i++) {
                // the +1 is because the first (0th) manager is the adaptor
                final int managerIndex = i + 1;
                assertTrue(poll(o -> managers[managerIndex].getRouter().allReachable("test-cluster3").size() == 1));
            }
            WordProducer.latch.countDown();
            final WordProducer adaptor = nodes.get(0).ctx.getBean(WordProducer.class);
            waitForAllSent(adaptor);
            // the last node has the RankCatcher
            final NodeManager manager = nodes.get(nodes.size() - 1).manager;
            final MessageProcessorLifecycle<?> mp = AccessUtil.getMp(manager, "test-cluster3");
            @SuppressWarnings("unchecked") final RankCatcher prototype = ((MessageProcessor<RankCatcher>) mp).getPrototype();
            if (strict) {
                final HashSet<String> expected = new HashSet<>(Arrays.asList("the", "that", "unto", "in", "and", "And", "of", "shall", "to", "he"));
                assertTrue(() -> {
                    return "FAILURE:" + expected.toString() + " != " + prototype.topRef.get();
                }, poll(prototype.topRef, tr -> {
                    final List<Rank> cur = tr.get();
                    final HashSet<String> topSet = new HashSet<>(cur.stream().map(r -> r.word).collect(Collectors.toSet()));
                    // once more than 1/2 of the list is there then we're done.
                    final int threshold = (expected.size() / 2) + 1;
                    int matches = 0;
                    for (final String exp : expected) if (topSet.contains(exp))
                        matches++;
                    return matches >= threshold;
                }));
            }
        });
    }
}