Examples with NodeStatsCollector net.dempsy.monitoring.NodeStatsCollector used on opensource projects

Example 1 with NodeStatsCollector

use of net.dempsy.monitoring.NodeStatsCollector in project Dempsy by Dempsy.

the class ApplicationState method apply.

public ApplicationState apply(final ApplicationState.Update update, final TransportManager tmanager, final NodeStatsCollector statsCollector, final RoutingStrategyManager manager, final String thisNodeId) {
    // apply toDelete first.
    final Set<NodeAddress> toDelete = update.toDelete;
    final boolean infoEnabled = LOGGER_SESSION.isInfoEnabled();
    if (toDelete.size() > 0) {
        // just clear all senders.
        if (infoEnabled)
            LOGGER_SESSION.info("[{}] Applying update to topology resulting in removing several destinations:", thisNodeId);
        for (final NodeAddress a : toDelete) {
            final Sender s = senders.remove(a);
            if (infoEnabled)
                LOGGER_SESSION.info("[{}]      removing sender ({}) to {}", thisNodeId, s, a);
            if (s != null)
                s.stop();
        }
    }
    final Map<NodeAddress, NodeInformation> newCurrent = new HashMap<>();
    // the one's to carry over.
    final Set<NodeInformation> leaveAlone = update.leaveAlone;
    if (leaveAlone.size() > 0)
        if (infoEnabled)
            LOGGER_SESSION.info("[{}] Applying update to topology resulting in leaving several destinations:", thisNodeId);
    for (final NodeInformation cur : leaveAlone) {
        if (infoEnabled)
            LOGGER_SESSION.info("[{}]      leaving alone : {}", thisNodeId, Optional.ofNullable(cur).map(ni -> ni.nodeAddress).orElse(null));
        newCurrent.put(cur.nodeAddress, cur);
    }
    // add new senders
    final Set<NodeInformation> toAdd = update.toAdd;
    if (toAdd.size() > 0)
        if (infoEnabled)
            LOGGER_SESSION.info("[{}] Applying update to topology resulting in adding several destinations:", thisNodeId);
    for (final NodeInformation cur : toAdd) {
        if (infoEnabled)
            LOGGER_SESSION.info("[{}]      adding : {}", thisNodeId, Optional.ofNullable(cur).map(ni -> ni.nodeAddress).orElse(null));
        newCurrent.put(cur.nodeAddress, cur);
    }
    // now flush out the remaining caches.
    // collapse all clusterInfos
    final Set<ClusterInformation> allCis = new HashSet<>();
    newCurrent.values().forEach(ni -> allCis.addAll(ni.clusterInfoByClusterId.values()));
    final Map<String, RoutingStrategy.Router> newOutboundByClusterName = new HashMap<>();
    final Map<String, Set<String>> cnByType = new HashMap<>();
    final Set<String> knownClusterOutbounds = new HashSet<>(outboundByClusterName_.keySet());
    for (final ClusterInformation ci : allCis) {
        final String clusterName = ci.clusterId.clusterName;
        final RoutingStrategy.Router ob = outboundByClusterName_.get(clusterName);
        knownClusterOutbounds.remove(clusterName);
        if (ob != null)
            newOutboundByClusterName.put(clusterName, ob);
        else {
            final RoutingStrategy.Factory obfactory = manager.getAssociatedInstance(ci.routingStrategyTypeId);
            final RoutingStrategy.Router nob = obfactory.getStrategy(ci.clusterId);
            newOutboundByClusterName.put(clusterName, nob);
        }
        // add all of the message types handled.
        ci.messageTypesHandled.forEach(mt -> {
            Set<String> entry = cnByType.get(mt);
            if (entry == null) {
                entry = new HashSet<>();
                cnByType.put(mt, entry);
            }
            entry.add(clusterName);
        });
    }
    return new ApplicationState(newOutboundByClusterName, cnByType, newCurrent, tmanager, thisNode, senders);
}

Example 2 with NodeStatsCollector

use of net.dempsy.monitoring.NodeStatsCollector in project Dempsy by Dempsy.

the class TestInstanceManager method setupContainer.

@SuppressWarnings("resource")
public Container setupContainer(final MessageProcessorLifecycle<?> prototype) throws ContainerException {
    dispatcher = new DummyDispatcher();
    statsCollector = new BasicClusterStatsCollector();
    manager = new NonLockingAltContainer().setMessageProcessor(prototype).setClusterId(new ClusterId("test", "test"));
    manager.setDispatcher(dispatcher);
    manager.setInbound(new DummyInbound());
    tm = new DefaultThreadingModel(TestInstanceManager.class.getName());
    tm.start(TestInstanceManager.class.getName());
    manager.start(new TestInfrastructure(tm) {

        BasicNodeStatsCollector nStats = new BasicNodeStatsCollector();

        @Override
        public ClusterStatsCollector getClusterStatsCollector(final ClusterId clusterId) {
            return statsCollector;
        }

        @Override
        public NodeStatsCollector getNodeStatsCollector() {
            return nStats;
        }
    });
    return manager;
}

Example 3 with NodeStatsCollector

use of net.dempsy.monitoring.NodeStatsCollector in project Dempsy by Dempsy.

the class TestContainerLoadHandling method setUp.

@Before
public void setUp() throws Exception {
    final ClusterId cid = new ClusterId("TestContainerLoadHandling", "test" + sequence++);
    dispatcher = new MockDispatcher();
    final BasicClusterStatsCollector sc = new BasicClusterStatsCollector();
    clusterStats = sc;
    nodeStats = new BasicNodeStatsCollector();
    container = tr.track(new Manager<Container>(Container.class).getAssociatedInstance(containerId)).setMessageProcessor(new MessageProcessor<TestMessageProcessor>(new TestMessageProcessor())).setClusterId(cid);
    container.setDispatcher(dispatcher);
    container.setInbound(new DummyInbound());
    container.start(new TestInfrastructure(null, null) {

        @Override
        public ClusterStatsCollector getClusterStatsCollector(final ClusterId clusterId) {
            return sc;
        }

        @Override
        public NodeStatsCollector getNodeStatsCollector() {
            return nodeStats;
        }
    });
    forceOutputException = false;
    stillRunning = true;
}

Example 4 with NodeStatsCollector

use of net.dempsy.monitoring.NodeStatsCollector in project Dempsy by Dempsy.

the class NodeManager method start.

public NodeManager start() throws DempsyException {
    validate();
    nodeStatsCollector = tr.track((NodeStatsCollector) node.getNodeStatsCollector());
    // TODO: cleaner?
    statsCollectorFactory = tr.track(new Manager<>(ClusterStatsCollectorFactory.class).getAssociatedInstance(node.getClusterStatsCollectorFactoryId()));
    // =====================================
    // set the dispatcher on adaptors and create containers for mp clusters
    final AtomicReference<String> firstAdaptorClusterName = new AtomicReference<>(null);
    node.getClusters().forEach(c -> {
        if (c.isAdaptor()) {
            if (firstAdaptorClusterName.get() == null)
                firstAdaptorClusterName.set(c.getClusterId().clusterName);
            final Adaptor adaptor = c.getAdaptor();
            adaptors.put(c.getClusterId(), adaptor);
            if (c.getRoutingStrategyId() != null && !"".equals(c.getRoutingStrategyId().trim()) && !" ".equals(c.getRoutingStrategyId().trim()))
                LOGGER.warn("The cluster " + c.getClusterId() + " contains an adaptor but also has the routingStrategy set. The routingStrategy will be ignored.");
            if (c.getOutputScheduler() != null)
                LOGGER.warn("The cluster " + c.getClusterId() + " contains an adaptor but also has an output executor set. The output executor will never be used.");
        } else {
            String containerTypeId = c.getContainerTypeId();
            if (containerTypeId == null)
                // can't be null
                containerTypeId = node.getContainerTypeId();
            final Container con = makeContainer(containerTypeId).setMessageProcessor(c.getMessageProcessor()).setClusterId(c.getClusterId()).setMaxPendingMessagesPerContainer(c.getMaxPendingMessagesPerContainer());
            // TODO: This is a hack for now.
            final Manager<RoutingStrategy.Inbound> inboundManager = new RoutingInboundManager();
            final RoutingStrategy.Inbound is = inboundManager.getAssociatedInstance(c.getRoutingStrategyId());
            final boolean outputSupported = c.getMessageProcessor().isOutputSupported();
            final Object outputScheduler = c.getOutputScheduler();
            // if there mp handles output but there's no output scheduler then we should warn.
            if (outputSupported && outputScheduler == null)
                LOGGER.warn("The cluster " + c.getClusterId() + " contains a message processor that supports an output cycle but there's no executor set so it will never be invoked.");
            if (!outputSupported && outputScheduler != null)
                LOGGER.warn("The cluster " + c.getClusterId() + " contains a message processor that doesn't support an output cycle but there's an output cycle executor set. The output cycle executor will never be used.");
            final OutputScheduler os = (outputSupported && outputScheduler != null) ? (OutputScheduler) outputScheduler : null;
            containers.add(new PerContainer(con, is, c, os));
        }
    });
    // it we're all adaptor then don't bother to get the receiver.
    if (containers.size() == 0) {
        // here there's no point in a receiver since there's nothing to receive.
        if (firstAdaptorClusterName.get() == null)
            throw new IllegalStateException("There seems to be no clusters or adaptors defined for this node \"" + node.toString() + "\"");
    } else {
        receiver = (Receiver) node.getReceiver();
        if (// otherwise we're all adaptor
        receiver != null)
            nodeAddress = receiver.getAddress(this);
        else if (firstAdaptorClusterName.get() == null)
            throw new IllegalStateException("There seems to be no clusters or adaptors defined for this node \"" + node.toString() + "\"");
    }
    nodeId = Optional.ofNullable(nodeAddress).map(n -> n.getGuid()).orElse(firstAdaptorClusterName.get());
    if (nodeStatsCollector == null) {
        LOGGER.warn("There is no {} set for the the application '{}'", StatsCollector.class.getSimpleName(), node.application);
        nodeStatsCollector = new DummyNodeStatsCollector();
    }
    nodeStatsCollector.setNodeId(nodeId);
    if (nodeAddress == null && node.getReceiver() != null)
        LOGGER.warn("The node at " + nodeId + " contains no message processors but has a Reciever set. The receiver will never be started.");
    if (nodeAddress == null && node.getDefaultRoutingStrategyId() != null)
        LOGGER.warn("The node at " + nodeId + " contains no message processors but has a defaultRoutingStrategyId set. The routingStrategyId will never be used.");
    if (threading == null)
        threading = tr.track(new DefaultThreadingModel(nodeId)).configure(node.getConfiguration()).start(nodeId);
    else if (!threading.isStarted())
        threading.start(nodeId);
    nodeStatsCollector.setMessagesPendingGauge(() -> threading.getNumberLimitedPending());
    final NodeReceiver nodeReciever = receiver == null ? null : tr.track(new NodeReceiver(containers.stream().map(pc -> pc.container).collect(Collectors.toList()), threading, nodeStatsCollector));
    final Map<ClusterId, ClusterInformation> messageTypesByClusterId = new HashMap<>();
    containers.stream().map(pc -> pc.clusterDefinition).forEach(c -> {
        messageTypesByClusterId.put(c.getClusterId(), new ClusterInformation(c.getRoutingStrategyId(), c.getClusterId(), c.getMessageProcessor().messagesTypesHandled()));
    });
    final NodeInformation nodeInfo = nodeAddress != null ? new NodeInformation(receiver.transportTypeId(), nodeAddress, messageTypesByClusterId) : null;
    // Then actually register the Node
    if (nodeInfo != null) {
        keepNodeRegstered = new PersistentTask(LOGGER, isRunning, persistenceScheduler, RETRY_PERIOND_MILLIS) {

            @Override
            public boolean execute() {
                try {
                    session.recursiveMkdir(rootPaths.clustersDir, null, DirMode.PERSISTENT, DirMode.PERSISTENT);
                    session.recursiveMkdir(rootPaths.nodesDir, null, DirMode.PERSISTENT, DirMode.PERSISTENT);
                    final String nodePath = rootPaths.nodesDir + "/" + nodeId;
                    session.mkdir(nodePath, nodeInfo, DirMode.EPHEMERAL);
                    final NodeInformation reread = (NodeInformation) session.getData(nodePath, this);
                    final boolean ret = nodeInfo.equals(reread);
                    if (ret == true)
                        ptaskReady.set(true);
                    return ret;
                } catch (final ClusterInfoException e) {
                    final String logmessage = "Failed to register the node. Retrying in " + RETRY_PERIOND_MILLIS + " milliseconds.";
                    if (LOGGER.isDebugEnabled())
                        LOGGER.info(logmessage, e);
                    else
                        LOGGER.info(logmessage, e);
                }
                return false;
            }

            @Override
            public String toString() {
                return "register node information";
            }
        };
    }
    // =====================================
    // The layering works this way.
    // 
    // Receiver -> NodeReceiver -> adaptor -> container -> OutgoingDispatcher -> RoutingStrategyOB -> Transport
    // 
    // starting needs to happen in reverse.
    // =====================================
    isRunning.set(true);
    this.tManager = tr.start(new TransportManager(), this);
    this.rsManager = tr.start(new RoutingStrategyManager(), this);
    // create the router but don't start it yet.
    this.router = new OutgoingDispatcher(rsManager, nodeAddress, nodeId, nodeReciever, tManager, nodeStatsCollector);
    // set up containers
    containers.forEach(pc -> pc.container.setDispatcher(router).setEvictionCycle(pc.clusterDefinition.getEvictionFrequency().evictionFrequency, pc.clusterDefinition.getEvictionFrequency().evictionTimeUnit));
    // IB routing strategy
    final int numContainers = containers.size();
    for (int i = 0; i < numContainers; i++) {
        final PerContainer c = containers.get(i);
        c.inboundStrategy.setContainerDetails(c.clusterDefinition.getClusterId(), new ContainerAddress(nodeAddress, i), c.container);
    }
    // setup the output executors by passing the containers
    containers.stream().filter(pc -> pc.outputScheduler != null).forEach(pc -> pc.outputScheduler.setOutputInvoker(pc.container));
    // set up adaptors
    adaptors.values().forEach(a -> a.setDispatcher(router));
    // start containers after setting inbound
    containers.forEach(pc -> tr.start(pc.container.setInbound(pc.inboundStrategy), this));
    // start the output schedulers now that the containers have been started.
    containers.stream().map(pc -> pc.outputScheduler).filter(os -> os != null).forEach(os -> tr.start(os, this));
    // start IB routing strategy
    containers.forEach(pc -> tr.start(pc.inboundStrategy, this));
    // start router
    tr.start(this.router, this);
    final PersistentTask startAdaptorAfterRouterIsRunning = new PersistentTask(LOGGER, isRunning, this.persistenceScheduler, 500) {

        @Override
        public boolean execute() {
            if (!router.isReady())
                return false;
            adaptors.entrySet().forEach(e -> threading.runDaemon(() -> tr.track(e.getValue()).start(), "Adaptor-" + e.getKey().clusterName));
            return true;
        }
    };
    // make sure the router is running. Once it is, start the adaptor(s)
    startAdaptorAfterRouterIsRunning.process();
    if (receiver != null)
        tr.track(receiver).start(nodeReciever, this);
    // close this session when we're done.
    tr.track(session);
    // make it known we're here and ready
    if (keepNodeRegstered != null)
        keepNodeRegstered.process();
    else
        ptaskReady.set(true);
    return this;
}

Example 5 with NodeStatsCollector

use of net.dempsy.monitoring.NodeStatsCollector in project Dempsy by Dempsy.

the class TestInstanceManager method setupContainer.

@SuppressWarnings("resource")
public LockingContainer setupContainer(final MessageProcessorLifecycle<?> prototype) throws ContainerException {
    dispatcher = new DummyDispatcher();
    statsCollector = new BasicClusterStatsCollector();
    nodeStats = new BasicNodeStatsCollector();
    container = (LockingContainer) new LockingContainer().setMessageProcessor(prototype).setClusterId(new ClusterId("test", "test"));
    container.setDispatcher(dispatcher);
    container.setInbound(new DummyInbound());
    tm = new DefaultThreadingModel(TestInstanceManager.class.getName());
    tm.start(TestInstanceManager.class.getName());
    container.start(new TestInfrastructure(tm) {

        @Override
        public BasicClusterStatsCollector getClusterStatsCollector(final ClusterId clusterId) {
            return statsCollector;
        }

        @Override
        public NodeStatsCollector getNodeStatsCollector() {
            return nodeStats;
        }
    });
    return container;
}