Examples with SystemPropertyManager net.dempsy.util.SystemPropertyManager used on opensource projects

Example 1 with SystemPropertyManager

use of net.dempsy.util.SystemPropertyManager in project Dempsy by Dempsy.

the class DempsyBaseTest method runCombos.

protected void runCombos(final String testName, final ComboFilter filter, final String[][] ctxs, final String[][][] perNodeProps, final TestToRun test) throws Exception {
    if (filter != null && !filter.filter(routerId, containerId, sessionType, transportType, serializerType))
        return;
    final long comboSequence = runComboSequence.getAndIncrement();
    currentAppName = testName + "-" + comboSequence;
    LOGGER.info("=====================================================================================");
    LOGGER.info("======== Running (" + comboSequence + ") " + testName + " with " + routerId + ", " + containerId + ", " + sessionType + ", " + threadingModelDescription + ", " + transportType + "/" + serializerType);
    try (final ServiceTracker tr = new ServiceTracker()) {
        currentlyTracking = tr;
        tr.track(new SystemPropertyManager()).set("routing-strategy", ROUTER_ID_PREFIX + routerId).set("container-type", CONTAINER_ID_PREFIX + containerId).set("test-name", currentAppName).setIfAbsent("total_shards", Integer.toString(NUM_MICROSHARDS));
        // instantiate session factory
        final ClusterInfoSessionFactory sessFact = tr.track(new ClassPathXmlApplicationContext(COLLAB_CTX_PREFIX + sessionType + COLLAB_CTX_SUFFIX)).getBean(ClusterInfoSessionFactory.class);
        currentSessionFactory = sessFact;
        final List<NodeManagerWithContext> reverseCpCtxs = reverseRange(0, ctxs.length).mapToObj(i -> {
            try (final SystemPropertyManager p2 = new SystemPropertyManager()) {
                if (perNodeProps != null && perNodeProps[i] != null) {
                    for (final String[] kv : perNodeProps[i]) {
                        if (kv != null) {
                            if (kv.length != 2)
                                throw new IllegalArgumentException("Invalid KV Pair passed for per-node property");
                            p2.set(kv[0], kv[1]);
                        }
                    }
                }
                final NodeManagerWithContext ret = makeNode(ctxs[i]);
                // we can only do this level of polling when the min_nodes isn't set or is set to 1.
                final String minNodesProp = System.getProperty("min_nodes");
                if (minNodesProp != null && Integer.parseInt(minNodesProp) == 1)
                    assertTrue(qpoll(ret, o -> o.manager.isReady()));
                return ret;
            }
        }).collect(Collectors.toList());
        final List<NodeManagerWithContext> cpCtxs = reverseRange(0, reverseCpCtxs.size()).mapToObj(i -> reverseCpCtxs.get(i)).collect(Collectors.toList());
        for (final NodeManagerWithContext n : cpCtxs) assertTrue(poll(o -> n.manager.isReady()));
        test.test(new Nodes(cpCtxs, sessFact));
        currentlyTracking = null;
    } finally {
        LocalClusterSessionFactory.completeReset();
        BlockingQueueAddress.completeReset();
        ClassTracker.dumpResults();
    }
    test.postShutdown();
    System.gc();
    System.gc();
    System.gc();
    System.gc();
    System.gc();
    System.gc();
    for (final long endTime = System.currentTimeMillis() + TEN_SECONDS; Thread.activeCount() > 3; ) {
        ignore(() -> Thread.sleep(10));
        if (Thread.activeCount() > 3 && (System.currentTimeMillis() > endTime)) {
            LOGGER.error("There appears to be an incomplete shutdown! There are {} threads running after stop.", Thread.activeCount());
            break;
        }
    }
}

Example 2 with SystemPropertyManager

use of net.dempsy.util.SystemPropertyManager in project Dempsy by Dempsy.

the class TestWordCount method testWordCountNoRankMultinode.

@Test
public void testWordCountNoRankMultinode() throws Throwable {
    try (@SuppressWarnings("resource") final SystemPropertyManager props = new SystemPropertyManager().set("min_nodes", "2")) {
        final String[][] ctxs = { { "classpath:/word-count/adaptor-kjv.xml", "classpath:/word-count/mp-word-count.xml" }, { "classpath:/word-count/mp-word-count.xml" } };
        WordCounter.aliveWordsToTestPassivate.clear();
        // need to make it wait.
        WordProducer.latch = new CountDownLatch(1);
        runCombos("testWordCountNoRankMultinode", (r, c, s, t, ser) -> isElasticRoutingStrategy(r) && isContainerOkay(c), ctxs, n -> {
            final ClusterId countCId = new ClusterId(currentAppName, "test-cluster1");
            final List<NodeManagerWithContext> nodes = n.nodes;
            final NodeManager[] manager = Arrays.asList(nodes.get(0).manager, nodes.get(1).manager).toArray(new NodeManager[2]);
            final ClassPathXmlApplicationContext[] ctx = Arrays.asList(nodes.get(0).ctx, nodes.get(1).ctx).toArray(new ClassPathXmlApplicationContext[2]);
            // wait until I can reach the cluster from the adaptor.
            assertTrue(poll(o -> manager[0].getRouter().allReachable("test-cluster1").size() == 2));
            assertTrue(poll(o -> manager[1].getRouter().allReachable("test-cluster1").size() == 2));
            WordProducer.latch.countDown();
            final WordProducer adaptor = ctx[0].getBean(WordProducer.class);
            assertTrue(waitForAllSent(adaptor));
            assertTrue(poll(o -> adaptor.done.get()));
            final var statsByNode = Arrays.asList(manager).stream().map(nm -> Pair.with((NodeMetricGetters) nm.getNodeStatsCollector(), (ClusterMetricGetters) nm.getClusterStatsCollector(countCId))).collect(Collectors.toList());
            assertTrue(poll(o -> {
                final int allDiscardedMessagesCount = allDiscardedMessagesCount(null, castToTupleList2(statsByNode));
                return adaptor.numDispatched == allDiscardedMessagesCount + statsByNode.stream().mapToInt(sc -> (int) sc.getValue1().getDispatchedMessageCount()).sum();
            }));
        });
        final int remaining = WordCounter.aliveWordsToTestPassivate.size();
        WordCounter.aliveWordsToTestPassivate.clear();
        assertEquals(0, remaining);
    }
}

Example 3 with SystemPropertyManager

use of net.dempsy.util.SystemPropertyManager in project Dempsy by Dempsy.

the class TestWordCount method testWordCountHomogeneousProcessing.

@Test
public void testWordCountHomogeneousProcessing() throws Throwable {
    final String[][] ctxs = { // adaptor only node
    { "classpath:/word-count/adaptor-kjv.xml" }, { "classpath:/word-count/mp-word-count.xml", "classpath:/word-count/mp-word-rank.xml" }, { "classpath:/word-count/mp-word-count.xml", "classpath:/word-count/mp-word-rank.xml" }, { "classpath:/word-count/mp-word-count.xml", "classpath:/word-count/mp-word-rank.xml" }, { "classpath:/word-count/mp-word-count.xml", "classpath:/word-count/mp-word-rank.xml" } };
    // the adaptor is the first one.
    final int NUM_WC = ctxs.length - 1;
    try (@SuppressWarnings("resource") final SystemPropertyManager props = new SystemPropertyManager().set("min_nodes", Integer.toString(NUM_WC)).set("routing-group", ":group").set("send_threads", "1").set("receive_threads", "1").set("blocking-queue-size", "500000")) {
        WordCounter.aliveWordsToTestPassivate.clear();
        // need to make it wait.
        WordProducer.latch = new CountDownLatch(1);
        runCombos("testWordCountHomogeneousProcessing", (r, c, s, t, ser) -> isElasticRoutingStrategy(r) && isContainerOkay(c), ctxs, n -> {
            final ClusterId countCId = new ClusterId(currentAppName, "test-cluster1");
            final ClusterId rankCId = new ClusterId(currentAppName, "test-cluster2");
            final List<NodeManagerWithContext> nodes = n.nodes;
            final NodeManager[] managers = nodes.stream().map(nm -> nm.manager).toArray(NodeManager[]::new);
            // wait until I can reach the cluster from the adaptor.
            assertTrue(poll(o -> managers[0].getRouter().allReachable("test-cluster1").size() == NUM_WC));
            assertTrue(poll(o -> managers[0].getRouter().allReachable("test-cluster2").size() == NUM_WC));
            WordProducer.latch.countDown();
            final WordProducer adaptor = nodes.get(0).ctx.getBean(WordProducer.class);
            waitForAllSent(adaptor);
            // get all of the stats collectors.
            final List<Triplet<NodeMetricGetters, ClusterMetricGetters, ClusterMetricGetters>> statsByNode = Arrays.asList(managers).subList(1, managers.length).stream().map(nm -> Triplet.with((NodeMetricGetters) nm.getNodeStatsCollector(), (ClusterMetricGetters) nm.getClusterStatsCollector(countCId), (ClusterMetricGetters) nm.getClusterStatsCollector(rankCId))).collect(Collectors.toList());
            final NodeMetricGetters adaptorStats = (NodeMetricGetters) managers[0].getNodeStatsCollector();
            final int totalSent = adaptor.numDispatched;
            // now wait for the sum of all messages received by the ranking to be the number sent
            assertTrue(poll(o -> {
                // System.out.println("" + adaptor.numDispatched + " ==? ");
                final int allDiscardedMessagesCount = allDiscardedMessagesCount(adaptorStats, castToTupleList3(statsByNode));
                final int totalRanked = allDiscardedMessagesCount + statsByNode.stream().mapToInt(sc -> (int) sc.getValue2().getDispatchedMessageCount()).sum();
                // System.out.println(": " + totalRanked + " == " + totalSent);
                return totalRanked == totalSent;
            }));
            // no nodes (except the adaptor node) should have sent any messages.
            // IOW, messages got to the Rank processor never leaving the node the Count was executed.
            final List<NodeMetricGetters> nodeStats = Arrays.asList(managers).subList(1, managers.length).stream().map(nm -> nm.getNodeStatsCollector()).map(s -> (NodeMetricGetters) s).collect(Collectors.toList());
            // if the routing id isn't a group id then there should be cross talk.
            assertEquals(NUM_WC, nodeStats.size());
            for (final NodeMetricGetters mg : nodeStats) assertEquals(0, mg.getMessagesNotSentCount());
            if (isGroupRoutingStrategy(routerId)) {
                for (final NodeMetricGetters mg : nodeStats) assertEquals(0, mg.getMessagesSentCount());
            } else {
                assertNotNull(nodeStats.stream().filter(mg -> mg.getMessagesSentCount() > 0).findFirst().orElse(null));
            }
        });
        final int remaining = WordCounter.aliveWordsToTestPassivate.size();
        WordCounter.aliveWordsToTestPassivate.clear();
        assertEquals(0, remaining);
    }
}

Example 4 with SystemPropertyManager

use of net.dempsy.util.SystemPropertyManager in project Dempsy by Dempsy.

the class TestWordCount method testWordCountNoRank.

// ========================================================================
@Test
public void testWordCountNoRank() throws Throwable {
    try (@SuppressWarnings("resource") final SystemPropertyManager props = new SystemPropertyManager().set("min_nodes", "1")) {
        final String[][] ctxs = { { "classpath:/word-count/adaptor-kjv.xml", "classpath:/word-count/mp-word-count.xml" } };
        WordCounter.aliveWordsToTestPassivate.clear();
        // need to make it wait.
        WordProducer.latch = new CountDownLatch(1);
        runCombos("testWordCountNoRank", (r, c, s, t, ser) -> isContainerOkay(c), ctxs, n -> {
            final List<NodeManagerWithContext> nodes = n.nodes;
            final NodeManager manager = nodes.get(0).manager;
            // wait until I can reach the cluster from the adaptor.
            assertTrue(poll(o -> manager.getRouter().allReachable("test-cluster1").size() == 1));
            final ClassPathXmlApplicationContext ctx = nodes.get(0).ctx;
            final WordProducer adaptor;
            final ClusterMetricGetters stats;
            WordProducer.latch.countDown();
            adaptor = ctx.getBean(WordProducer.class);
            stats = (ClusterMetricGetters) manager.getClusterStatsCollector(new ClusterId(currentAppName, "test-cluster1"));
            assertTrue(poll(o -> adaptor.done.get()));
            assertTrue(poll(o -> {
                // System.out.println("" + adaptor.numDispatched + " == " + stats.getProcessedMessageCount());
                return adaptor.numDispatched == (stats.getProcessedMessageCount() + stats.getMessageDiscardedCount());
            }));
        });
        final int remaining = WordCounter.aliveWordsToTestPassivate.size();
        WordCounter.aliveWordsToTestPassivate.clear();
        assertEquals(0, remaining);
    }
}

Example 5 with SystemPropertyManager

use of net.dempsy.util.SystemPropertyManager in project Dempsy by Dempsy.

the class BlockingQueueTest method testBlockingQueueOverflow.

/**
 * Test overflow for a blocking Transport around a queue with depth one. While the transport will not call the, and does not even have a , overflow handler,
 * every message will call the overflow handler on
 * the receiver since the queue is always full.
 *
 * @throws Throwable
 */
@Test
public void testBlockingQueueOverflow() throws Throwable {
    final AtomicReference<String> message = new AtomicReference<String>(null);
    final ArrayBlockingQueue<Object> input = new ArrayBlockingQueue<>(1);
    try (@SuppressWarnings("resource") SystemPropertyManager // test only works when the blocking queue blocks
    props = new SystemPropertyManager().set(BlockingQueueSenderFactory.class.getPackageName() + "." + BlockingQueueSenderFactory.BLOCKING_KEY, "true");
        final TestInfrastructure infra = new TestInfrastructure(new DefaultThreadingModel("BQTest-testBlockingQueueOverflow-"));
        final Receiver r = new BlockingQueueReceiver(input);
        final TransportManager tranMan = chain(new TransportManager(), c -> c.start(infra));
        final SenderFactory sf = tranMan.getAssociatedInstance(transportTypeId)) {
        final Sender sender = sf.getSender(r.getAddress(infra));
        final AtomicBoolean finallySent = new AtomicBoolean(false);
        final AtomicLong receiveCount = new AtomicLong();
        // fill up queue
        sender.send("Hello");
        final Thread t = new Thread(() -> {
            try {
                sender.send("Hello again");
            } catch (final MessageTransportException | InterruptedException e) {
                throw new RuntimeException(e);
            }
            finallySent.set(true);
        });
        t.start();
        Thread.sleep(100);
        // the thread should be hung blocked on the send
        assertFalse(finallySent.get());
        // Start the receiver to read
        r.start((final String msg) -> {
            message.set(new String(msg));
            receiveCount.incrementAndGet();
            return true;
        }, infra);
        // 2 messages should have been read and the 2nd should be "Hello again"
        assertTrue(poll(o -> "Hello again".equals(message.get())));
        // The thread should shut down eventually
        assertTrue(poll(o -> !t.isAlive()));
    }
}