Examples with Receiver loghub.Receiver used on opensource projects

Example 1 with Receiver

use of loghub.Receiver in project LogHub by fbacchella.

the class Configuration method analyze.

private Properties analyze(ConfigListener conf) throws ConfigException {
    final Map<String, Object> newProperties = new HashMap<String, Object>(conf.properties.size() + Properties.PROPSNAMES.values().length + System.getProperties().size());
    // Resolvers properties found and and it to new properties
    Function<Object, Object> resolve = i -> {
        return ((i instanceof ConfigListener.ObjectWrapped) ? ((ConfigListener.ObjectWrapped) i).wrapped : (i instanceof ConfigListener.ObjectReference) ? parseObjectDescription((ConfigListener.ObjectDescription) i, emptyConstructor) : i);
    };
    conf.properties.entrySet().stream().forEach(i -> newProperties.put(i.getKey(), resolve.apply(i.getValue())));
    // Resolve the sources
    ThrowingFunction<Class<Source>, Source> sourceConstructor = i -> {
        return i.getConstructor().newInstance();
    };
    conf.sources.forEach((name, sd) -> {
        Source s = parseObjectDescription(sd, sourceConstructor);
        s.setName(name);
        sources.put(name, s);
    });
    Map<String, Pipeline> namedPipeLine = new HashMap<>(conf.pipelines.size());
    newProperties.put(Properties.PROPSNAMES.CLASSLOADERNAME.toString(), classLoader);
    Set<Pipeline> pipelines = new HashSet<>();
    // Generate all the named pipeline
    for (Entry<String, ConfigListener.PipenodesList> e : conf.pipelines.entrySet()) {
        String name = e.getKey();
        Pipeline p = parsePipeline(e.getValue(), name, 0, new AtomicInteger());
        pipelines.add(p);
        namedPipeLine.put(name, p);
        if (p.nextPipeline != null) {
            topPipelines.add(p.nextPipeline);
        }
    }
    newProperties.put(Properties.PROPSNAMES.PIPELINES.toString(), Collections.unmodifiableSet(pipelines));
    namedPipeLine = Collections.unmodifiableMap(namedPipeLine);
    newProperties.put(Properties.PROPSNAMES.NAMEDPIPELINES.toString(), namedPipeLine);
    // Find the queue depth
    final int queuesDepth = newProperties.containsKey("queueDepth") ? (Integer) newProperties.remove("queueDepth") : DEFAULTQUEUEDEPTH;
    newProperties.put(Properties.PROPSNAMES.QUEUESDEPTH.toString(), queuesDepth);
    BlockingQueue<Event> mainQueue = new ArrayBlockingQueue<Event>(queuesDepth);
    Map<String, BlockingQueue<Event>> outputQueues = new HashMap<>(namedPipeLine.size());
    namedPipeLine.keySet().stream().forEach(i -> outputQueues.put(i, new ArrayBlockingQueue<Event>(queuesDepth)));
    newProperties.put(Properties.PROPSNAMES.FORMATTERS.toString(), conf.formatters);
    newProperties.put(Properties.PROPSNAMES.MAINQUEUE.toString(), mainQueue);
    newProperties.put(Properties.PROPSNAMES.OUTPUTQUEUE.toString(), outputQueues);
    // Fill the receivers list
    receivers = new ArrayList<>();
    for (Input i : conf.inputs) {
        if (i.piperef == null || !namedPipeLine.containsKey(i.piperef)) {
            throw new RuntimeException("Invalid input, no destination pipeline: " + i);
        }
        for (ConfigListener.ObjectDescription desc : i.receiver) {
            Pipeline p = namedPipeLine.get(i.piperef);
            ThrowingFunction<Class<Receiver>, Receiver> receiverConstructor = r -> {
                return r.getConstructor(BlockingQueue.class, Pipeline.class).newInstance(mainQueue, p);
            };
            Receiver r = (Receiver) parseObjectDescription(desc, receiverConstructor);
            receivers.add(r);
        }
        inputpipelines.add(i.piperef);
    }
    topPipelines.addAll(inputpipelines);
    inputpipelines = Collections.unmodifiableSet(inputpipelines);
    receivers = Collections.unmodifiableList(receivers);
    newProperties.put(Properties.PROPSNAMES.RECEIVERS.toString(), receivers);
    // Fill the senders list
    senders = new ArrayList<>();
    for (Output o : conf.outputs) {
        if (o.piperef == null || !namedPipeLine.containsKey(o.piperef)) {
            throw new RuntimeException("Invalid output, no source pipeline: " + o);
        }
        for (ConfigListener.ObjectDescription desc : o.sender) {
            BlockingQueue<Event> out = outputQueues.get(o.piperef);
            ThrowingFunction<Class<Sender>, Sender> senderConstructor = r -> {
                return r.getConstructor(BlockingQueue.class).newInstance(out);
            };
            Sender s = (Sender) parseObjectDescription(desc, senderConstructor);
            // logger.debug("sender {} source point will be {}", () -> s, () -> namedPipeLine.get(o.piperef).outQueue);
            senders.add(s);
        }
        outputpipelines.add(o.piperef);
    }
    topPipelines.addAll(outputpipelines);
    outputpipelines = Collections.unmodifiableSet(outputpipelines);
    senders = Collections.unmodifiableList(senders);
    newProperties.put(Properties.PROPSNAMES.SENDERS.toString(), senders);
    newProperties.put(Properties.PROPSNAMES.TOPPIPELINE.toString(), Collections.unmodifiableSet(topPipelines));
    newProperties.put(Properties.PROPSNAMES.SOURCES.toString(), Collections.unmodifiableMap(sources));
    // Allows the system properties to override any properties given in the configuration file
    // But only if they are not some of the special internal properties
    Set<String> privatepropsnames = new HashSet<>(Properties.PROPSNAMES.values().length);
    Arrays.stream(Properties.PROPSNAMES.values()).forEach(i -> privatepropsnames.add(i.toString()));
    ;
    System.getProperties().entrySet().stream().filter(i -> !privatepropsnames.contains(i.getKey())).forEach(i -> newProperties.put(i.getKey().toString(), i.getValue()));
    return new Properties(newProperties);
}

Example 2 with Receiver

use of loghub.Receiver in project LogHub by fbacchella.

the class TestWithZMQ method testSimpleInput.

@Test(timeout = 3000)
public void testSimpleInput() throws InterruptedException, ConfigException, IOException {
    Properties conf = Tools.loadConf("simpleinput.conf");
    logger.debug("pipelines: {}", conf.pipelines);
    for (Receiver r : conf.receivers) {
        Assert.assertTrue("failed to configure " + r, r.configure(conf));
        r.start();
    }
    for (Sender s : conf.senders) {
        Assert.assertTrue("failed to configure " + s, s.configure(conf));
        s.start();
    }
    Socket out = tctxt.ctx.newSocket(Method.CONNECT, Type.SUB, "inproc://sender", 1, -1);
    out.subscribe(new byte[] {});
    Socket sender = tctxt.ctx.newSocket(Method.CONNECT, Type.PUB, "inproc://listener", 1, -1);
    // Wait for ZMQ to be started
    Thread.sleep(30);
    sender.send("something");
    Event received = conf.mainQueue.poll(1, TimeUnit.SECONDS);
    Assert.assertNotNull("nothing received", received);
    conf.outputQueues.get("main").add(received);
    byte[] buffer = out.recv();
    Assert.assertEquals("wrong send message", "something", new String(buffer));
    tctxt.ctx.close(sender);
    tctxt.ctx.close(out);
    for (Receiver r : conf.receivers) {
        r.interrupt();
    }
    for (Sender s : conf.senders) {
        s.interrupt();
    }
    SmartContext.getContext().terminate();
}