Examples with Adaptor net.dempsy.messages.Adaptor used on opensource projects

Example 1 with Adaptor

use of net.dempsy.messages.Adaptor in project Dempsy by Dempsy.

the class TestWordCount method testWordCountHomogeneousProcessing.

@Test
public void testWordCountHomogeneousProcessing() throws Throwable {
    final String[][] ctxs = { // adaptor only node
    { "classpath:/word-count/adaptor-kjv.xml" }, { "classpath:/word-count/mp-word-count.xml", "classpath:/word-count/mp-word-rank.xml" }, { "classpath:/word-count/mp-word-count.xml", "classpath:/word-count/mp-word-rank.xml" }, { "classpath:/word-count/mp-word-count.xml", "classpath:/word-count/mp-word-rank.xml" }, { "classpath:/word-count/mp-word-count.xml", "classpath:/word-count/mp-word-rank.xml" } };
    // the adaptor is the first one.
    final int NUM_WC = ctxs.length - 1;
    try (@SuppressWarnings("resource") final SystemPropertyManager props = new SystemPropertyManager().set("min_nodes", Integer.toString(NUM_WC)).set("routing-group", ":group").set("send_threads", "1").set("receive_threads", "1").set("blocking-queue-size", "500000")) {
        // need to make it wait.
        WordProducer.latch = new CountDownLatch(1);
        runCombos("testWordCountHomogeneousProcessing", (r, c, s, t, ser) -> isElasticRoutingStrategy(r), ctxs, n -> {
            final List<NodeManagerWithContext> nodes = n.nodes;
            final NodeManager[] managers = nodes.stream().map(nm -> nm.manager).toArray(NodeManager[]::new);
            // wait until I can reach the cluster from the adaptor.
            assertTrue(poll(o -> managers[0].getRouter().allReachable("test-cluster1").size() == NUM_WC));
            assertTrue(poll(o -> managers[0].getRouter().allReachable("test-cluster2").size() == NUM_WC));
            WordProducer.latch.countDown();
            final WordProducer adaptor = nodes.get(0).ctx.getBean(WordProducer.class);
            waitForAllSent(adaptor);
            // get all of the stats collectors for the ranks.
            final List<ClusterMetricGetters> rankStats = Arrays.asList(managers).subList(1, managers.length).stream().map(nm -> nm.getClusterStatsCollector(new ClusterId(currentAppName, "test-cluster2"))).map(sc -> (ClusterMetricGetters) sc).collect(Collectors.toList());
            final int totalSent = adaptor.numDispatched;
            // now wait for the sum of all messages received by the ranking to be the number sent
            assertTrue(poll(o -> {
                final int totalRanked = rankStats.stream().map(sc -> Integer.valueOf((int) sc.getDispatchedMessageCount())).reduce(Integer.valueOf(0), (i1, i2) -> new Integer(i1.intValue() + i2.intValue())).intValue();
                return totalRanked == totalSent;
            }));
            // no nodes (except the adaptor node) should have sent any messages.
            // IOW, messages got to the Rank processor never leaving the node the Count was executed.
            final List<NodeMetricGetters> nodeStats = Arrays.asList(managers).subList(1, managers.length).stream().map(nm -> nm.getNodeStatsCollector()).map(s -> (NodeMetricGetters) s).collect(Collectors.toList());
            // if the routing id isn't a group id then there should be cross talk.
            assertEquals(NUM_WC, nodeStats.size());
            for (final NodeMetricGetters mg : nodeStats) assertEquals(0, mg.getMessagesNotSentCount());
            if (isGroupRoutingStrategy(routerId)) {
                for (final NodeMetricGetters mg : nodeStats) assertEquals(0, mg.getMessagesSentCount());
            } else {
                assertNotNull(nodeStats.stream().filter(mg -> mg.getMessagesSentCount() > 0).findFirst().orElse(null));
            }
        });
    }
}

Example 2 with Adaptor

use of net.dempsy.messages.Adaptor in project Dempsy by Dempsy.

the class TestWordCount method testWordCountNoRankMultinode.

@Test
public void testWordCountNoRankMultinode() throws Throwable {
    try (@SuppressWarnings("resource") final SystemPropertyManager props = new SystemPropertyManager().set("min_nodes", "2")) {
        final String[][] ctxs = { { "classpath:/word-count/adaptor-kjv.xml", "classpath:/word-count/mp-word-count.xml" }, { "classpath:/word-count/mp-word-count.xml" } };
        // need to make it wait.
        WordProducer.latch = new CountDownLatch(1);
        runCombos("testWordCountNoRankMultinode", (r, c, s, t, ser) -> isElasticRoutingStrategy(r), ctxs, n -> {
            final List<NodeManagerWithContext> nodes = n.nodes;
            final NodeManager[] manager = Arrays.asList(nodes.get(0).manager, nodes.get(1).manager).toArray(new NodeManager[2]);
            final ClassPathXmlApplicationContext[] ctx = Arrays.asList(nodes.get(0).ctx, nodes.get(1).ctx).toArray(new ClassPathXmlApplicationContext[2]);
            // wait until I can reach the cluster from the adaptor.
            assertTrue(poll(o -> manager[0].getRouter().allReachable("test-cluster1").size() == 2));
            assertTrue(poll(o -> manager[1].getRouter().allReachable("test-cluster1").size() == 2));
            WordProducer.latch.countDown();
            final WordProducer adaptor = ctx[0].getBean(WordProducer.class);
            final ClusterMetricGetters[] stats = Arrays.asList((ClusterMetricGetters) manager[0].getClusterStatsCollector(new ClusterId(currentAppName, "test-cluster1")), (ClusterMetricGetters) manager[1].getClusterStatsCollector(new ClusterId(currentAppName, "test-cluster1"))).toArray(new ClusterMetricGetters[2]);
            assertTrue(waitForAllSent(adaptor));
            assertTrue(poll(o -> {
                // System.out.println(stats[0].getProcessedMessageCount() + ", " + stats[1].getProcessedMessageCount());
                return adaptor.numDispatched == Arrays.stream(stats).map(c -> c.getProcessedMessageCount()).reduce((c1, c2) -> c1.longValue() + c2.longValue()).get().longValue();
            }));
        });
    }
}

Example 3 with Adaptor

use of net.dempsy.messages.Adaptor in project Dempsy by Dempsy.

the class TestWordCount method testWordCountNoRank.

@Test
public void testWordCountNoRank() throws Throwable {
    try (@SuppressWarnings("resource") final SystemPropertyManager props = new SystemPropertyManager().set("min_nodes", "1")) {
        final String[][] ctxs = { { "classpath:/word-count/adaptor-kjv.xml", "classpath:/word-count/mp-word-count.xml" } };
        // need to make it wait.
        WordProducer.latch = new CountDownLatch(1);
        runCombos("testWordCountNoRank", ctxs, n -> {
            final List<NodeManagerWithContext> nodes = n.nodes;
            final NodeManager manager = nodes.get(0).manager;
            // wait until I can reach the cluster from the adaptor.
            assertTrue(poll(o -> manager.getRouter().allReachable("test-cluster1").size() == 1));
            final ClassPathXmlApplicationContext ctx = nodes.get(0).ctx;
            final WordProducer adaptor;
            final ClusterMetricGetters stats;
            WordProducer.latch.countDown();
            adaptor = ctx.getBean(WordProducer.class);
            stats = (ClusterMetricGetters) manager.getClusterStatsCollector(new ClusterId(currentAppName, "test-cluster1"));
            assertTrue(poll(o -> adaptor.done.get()));
            assertTrue(poll(o -> {
                // System.out.println("" + adaptor.numDispatched + " == " + stats.getProcessedMessageCount());
                return adaptor.numDispatched == stats.getProcessedMessageCount();
            }));
        });
    }
}

Example 4 with Adaptor

use of net.dempsy.messages.Adaptor in project Dempsy by Dempsy.

the class TestContainer method testInvokeOutput.

@Test
public void testInvokeOutput() throws Exception {
    outputMessages = Collections.newSetFromMap(new ConcurrentHashMap<>());
    cache = new ConcurrentHashMap<>();
    final TestAdaptor adaptor = context.getBean(TestAdaptor.class);
    assertNotNull(adaptor.dispatcher);
    adaptor.dispatcher.dispatchAnnotated(new ContainerTestMessage("foo"));
    adaptor.dispatcher.dispatchAnnotated(new ContainerTestMessage("bar"));
    assertTrue(poll(container, c -> c.getProcessorCount() > 1));
    Thread.sleep(100);
    assertEquals("number of MP instances", 2, container.getProcessorCount());
    try (NodeManager nman = addOutputCatchStage()) {
        final TestProcessor mp = cache.get("foo");
        assertTrue(poll(mp, m -> mp.invocationCount > 0));
        Thread.sleep(100);
        assertEquals("invocation count, 1st message", 1, mp.invocationCount);
        // because the sessionFactory is shared and the appname is the same, we should be in the same app
        container.outputPass();
        assertTrue(poll(outputMessages, o -> o.size() > 1));
        Thread.sleep(100);
        assertEquals(2, outputMessages.size());
        // no new mps created in the first one
        assertEquals("did not create MP", 2, container.getProcessorCount());
        // but the invocation count should have increased since the output cycles feeds messages back to this cluster
        assertTrue(poll(mp, m -> mp.invocationCount > 1));
        Thread.sleep(100);
        assertEquals("invocation count, 1st message", 2, mp.invocationCount);
        // // order of messages is not guaranteed, so we need to aggregate keys
        final HashSet<String> messageKeys = new HashSet<String>();
        final Iterator<OutputMessage> iter = outputMessages.iterator();
        messageKeys.add(iter.next().getKey());
        messageKeys.add(iter.next().getKey());
        assertTrue("first MP sent output", messageKeys.contains("foo"));
        assertTrue("second MP sent output", messageKeys.contains("bar"));
    }
}

Example 5 with Adaptor

use of net.dempsy.messages.Adaptor in project Dempsy by Dempsy.

the class TestContainer method testEvictableWithPassivateException.

@Test
public void testEvictableWithPassivateException() throws Exception {
    final TestProcessor mp = createAndGet("foo");
    mp.throwPassivateException.set(true);
    final TestProcessor prototype = context.getBean(TestProcessor.class);
    final int tmpCloneCount = prototype.cloneCount.intValue();
    mp.evict.set(true);
    container.evict();
    assertTrue(poll(o -> mp.passivateExceptionCount.get() > 0));
    Thread.sleep(100);
    assertEquals("Passivate Exception Thrown", 1, mp.passivateExceptionCount.get());
    final TestAdaptor adaptor = context.getBean(TestAdaptor.class);
    adaptor.dispatcher.dispatchAnnotated(new ContainerTestMessage("foo"));
    assertTrue(poll(o -> prototype.cloneCount.intValue() > tmpCloneCount));
    Thread.sleep(1000);
    assertEquals("Clone count, 2nd message", tmpCloneCount + 1, prototype.cloneCount.intValue());
}