Examples with Pipeline loghub.Pipeline used on opensource projects

Example 1 with Pipeline

use of loghub.Pipeline in project LogHub by fbacchella.

the class TestFire method test.

@Test
public void test() throws ProcessorException, InterruptedException, ConfigException, IOException {
    Properties conf = Tools.loadConf("fire.conf");
    for (Pipeline pipe : conf.pipelines) {
        Assert.assertTrue("configuration failed", pipe.configure(conf));
    }
    Event sent = Tools.getEvent();
    sent.put("count", 2);
    Tools.runProcessing(sent, conf.namedPipeLine.get("main"), conf);
    Event old = conf.mainQueue.remove();
    Event newevent = conf.mainQueue.remove();
    Assert.assertEquals("Not matching old event", old.get("count"), 2);
    Assert.assertEquals("Event not fired", 6, newevent.get("c"));
}

Example 2 with Pipeline

use of loghub.Pipeline in project LogHub by fbacchella.

the class Configuration method parsePipeline.

private Pipeline parsePipeline(ConfigListener.PipenodesList desc, String currentPipeLineName, int depth, AtomicInteger subPipeCount) throws ConfigException {
    List<Processor> allSteps = new ArrayList<Processor>() {

        @Override
        public String toString() {
            StringBuilder buffer = new StringBuilder();
            buffer.append("PipeList(");
            for (Processor i : this) {
                buffer.append(i);
                buffer.append(", ");
            }
            buffer.setLength(buffer.length() - 2);
            buffer.append(')');
            return buffer.toString();
        }
    };
    desc.processors.stream().map(i -> {
        return getProcessor(i, currentPipeLineName, depth, subPipeCount);
    }).forEach(allSteps::add);
    Pipeline pipe = new Pipeline(allSteps, currentPipeLineName + (depth == 0 ? "" : "$" + subPipeCount.getAndIncrement()), desc.nextPipelineName);
    return pipe;
}

Example 3 with Pipeline

use of loghub.Pipeline in project LogHub by fbacchella.

the class Configuration method analyze.

private Properties analyze(ConfigListener conf) throws ConfigException {
    final Map<String, Object> newProperties = new HashMap<String, Object>(conf.properties.size() + Properties.PROPSNAMES.values().length + System.getProperties().size());
    // Resolvers properties found and and it to new properties
    Function<Object, Object> resolve = i -> {
        return ((i instanceof ConfigListener.ObjectWrapped) ? ((ConfigListener.ObjectWrapped) i).wrapped : (i instanceof ConfigListener.ObjectReference) ? parseObjectDescription((ConfigListener.ObjectDescription) i, emptyConstructor) : i);
    };
    conf.properties.entrySet().stream().forEach(i -> newProperties.put(i.getKey(), resolve.apply(i.getValue())));
    // Resolve the sources
    ThrowingFunction<Class<Source>, Source> sourceConstructor = i -> {
        return i.getConstructor().newInstance();
    };
    conf.sources.forEach((name, sd) -> {
        Source s = parseObjectDescription(sd, sourceConstructor);
        s.setName(name);
        sources.put(name, s);
    });
    Map<String, Pipeline> namedPipeLine = new HashMap<>(conf.pipelines.size());
    newProperties.put(Properties.PROPSNAMES.CLASSLOADERNAME.toString(), classLoader);
    Set<Pipeline> pipelines = new HashSet<>();
    // Generate all the named pipeline
    for (Entry<String, ConfigListener.PipenodesList> e : conf.pipelines.entrySet()) {
        String name = e.getKey();
        Pipeline p = parsePipeline(e.getValue(), name, 0, new AtomicInteger());
        pipelines.add(p);
        namedPipeLine.put(name, p);
        if (p.nextPipeline != null) {
            topPipelines.add(p.nextPipeline);
        }
    }
    newProperties.put(Properties.PROPSNAMES.PIPELINES.toString(), Collections.unmodifiableSet(pipelines));
    namedPipeLine = Collections.unmodifiableMap(namedPipeLine);
    newProperties.put(Properties.PROPSNAMES.NAMEDPIPELINES.toString(), namedPipeLine);
    // Find the queue depth
    final int queuesDepth = newProperties.containsKey("queueDepth") ? (Integer) newProperties.remove("queueDepth") : DEFAULTQUEUEDEPTH;
    newProperties.put(Properties.PROPSNAMES.QUEUESDEPTH.toString(), queuesDepth);
    BlockingQueue<Event> mainQueue = new ArrayBlockingQueue<Event>(queuesDepth);
    Map<String, BlockingQueue<Event>> outputQueues = new HashMap<>(namedPipeLine.size());
    namedPipeLine.keySet().stream().forEach(i -> outputQueues.put(i, new ArrayBlockingQueue<Event>(queuesDepth)));
    newProperties.put(Properties.PROPSNAMES.FORMATTERS.toString(), conf.formatters);
    newProperties.put(Properties.PROPSNAMES.MAINQUEUE.toString(), mainQueue);
    newProperties.put(Properties.PROPSNAMES.OUTPUTQUEUE.toString(), outputQueues);
    // Fill the receivers list
    receivers = new ArrayList<>();
    for (Input i : conf.inputs) {
        if (i.piperef == null || !namedPipeLine.containsKey(i.piperef)) {
            throw new RuntimeException("Invalid input, no destination pipeline: " + i);
        }
        for (ConfigListener.ObjectDescription desc : i.receiver) {
            Pipeline p = namedPipeLine.get(i.piperef);
            ThrowingFunction<Class<Receiver>, Receiver> receiverConstructor = r -> {
                return r.getConstructor(BlockingQueue.class, Pipeline.class).newInstance(mainQueue, p);
            };
            Receiver r = (Receiver) parseObjectDescription(desc, receiverConstructor);
            receivers.add(r);
        }
        inputpipelines.add(i.piperef);
    }
    topPipelines.addAll(inputpipelines);
    inputpipelines = Collections.unmodifiableSet(inputpipelines);
    receivers = Collections.unmodifiableList(receivers);
    newProperties.put(Properties.PROPSNAMES.RECEIVERS.toString(), receivers);
    // Fill the senders list
    senders = new ArrayList<>();
    for (Output o : conf.outputs) {
        if (o.piperef == null || !namedPipeLine.containsKey(o.piperef)) {
            throw new RuntimeException("Invalid output, no source pipeline: " + o);
        }
        for (ConfigListener.ObjectDescription desc : o.sender) {
            BlockingQueue<Event> out = outputQueues.get(o.piperef);
            ThrowingFunction<Class<Sender>, Sender> senderConstructor = r -> {
                return r.getConstructor(BlockingQueue.class).newInstance(out);
            };
            Sender s = (Sender) parseObjectDescription(desc, senderConstructor);
            // logger.debug("sender {} source point will be {}", () -> s, () -> namedPipeLine.get(o.piperef).outQueue);
            senders.add(s);
        }
        outputpipelines.add(o.piperef);
    }
    topPipelines.addAll(outputpipelines);
    outputpipelines = Collections.unmodifiableSet(outputpipelines);
    senders = Collections.unmodifiableList(senders);
    newProperties.put(Properties.PROPSNAMES.SENDERS.toString(), senders);
    newProperties.put(Properties.PROPSNAMES.TOPPIPELINE.toString(), Collections.unmodifiableSet(topPipelines));
    newProperties.put(Properties.PROPSNAMES.SOURCES.toString(), Collections.unmodifiableMap(sources));
    // Allows the system properties to override any properties given in the configuration file
    // But only if they are not some of the special internal properties
    Set<String> privatepropsnames = new HashSet<>(Properties.PROPSNAMES.values().length);
    Arrays.stream(Properties.PROPSNAMES.values()).forEach(i -> privatepropsnames.add(i.toString()));
    ;
    System.getProperties().entrySet().stream().filter(i -> !privatepropsnames.contains(i.getKey())).forEach(i -> newProperties.put(i.getKey().toString(), i.getValue()));
    return new Properties(newProperties);
}

Example 4 with Pipeline

use of loghub.Pipeline in project LogHub by fbacchella.

the class TestServer method testSimple.

@Test(timeout = 2000)
public void testSimple() throws InterruptedException {
    Properties empty = new Properties(Collections.emptyMap());
    BlockingQueue<Event> receiver = new ArrayBlockingQueue<>(1);
    TesterReceiver r = new TesterReceiver(receiver, new Pipeline(Collections.emptyList(), "testone", null));
    r.configure(empty);
    final ChannelFuture[] sent = new ChannelFuture[1];
    EventLoopGroup workerGroup = new DefaultEventLoopGroup();
    Bootstrap b = new Bootstrap();
    b.group(workerGroup);
    b.channel(LocalChannel.class);
    b.handler(new SimpleChannelInboundHandler<ByteBuf>() {

        @Override
        public void channelActive(ChannelHandlerContext ctx) {
            sent[0] = ctx.writeAndFlush(Unpooled.copiedBuffer("Message\r\n", CharsetUtil.UTF_8));
        }

        @Override
        protected void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) throws Exception {
        }
    });
    // Start the client.
    ChannelFuture f = b.connect(new LocalAddress(TestServer.class.getCanonicalName())).sync();
    Thread.sleep(100);
    sent[0].sync();
    f.channel().close();
    // Wait until the connection is closed.
    f.channel().closeFuture().sync();
    Event e = receiver.poll();
    Assert.assertEquals("Message", e.get("message"));
}

Example 5 with Pipeline

use of loghub.Pipeline in project LogHub by fbacchella.

the class TestHttp method makeReceiver.

public void makeReceiver(Consumer<Http> prepare) throws IOException {
    // Generate a locally binded random socket
    ServerSocket socket = new ServerSocket(0, 10, InetAddress.getLoopbackAddress());
    hostname = socket.getInetAddress().getHostAddress();
    port = socket.getLocalPort();
    socket.close();
    queue = new ArrayBlockingQueue<>(1);
    receiver = new Http(queue, new Pipeline(Collections.emptyList(), "testhttp", null));
    receiver.setHost(hostname);
    receiver.setPort(port);
    receiver.setDecoder(new StringCodec());
    prepare.accept(receiver);
    Assert.assertTrue(receiver.configure(new Properties(Collections.emptyMap())));
    receiver.start();
}