Examples with Event loghub.Event used on opensource projects

Example 16 with Event

use of loghub.Event in project LogHub by fbacchella.

the class TestFileMap method testone.

@Test
public void testone() throws ProcessorException {
    URL ifpixurl = getClass().getResource("/ipfix-information-elements.csv");
    FileMap s = new FileMap();
    s.setMappingFile(ifpixurl.getFile());
    s.setCsvFormat("RFC4180");
    s.setKeyName("ElementID");
    s.setValueName("Name");
    s.configure(null);
    Mapper p = new Mapper();
    p.setField(new String[] { "type" });
    p.setLvalue(new String[] { "type" });
    p.setMap(s);
    Event e = Tools.getEvent();
    e.put("type", "1");
    ProcessingStatus ps = Tools.runProcessing(e, "main", Collections.singletonList(p));
    Event ep = ps.mainQueue.remove();
    Assert.assertEquals("octetDeltaCount", ep.get("type"));
}

Example 17 with Event

use of loghub.Event in project LogHub by fbacchella.

the class Configuration method analyze.

private Properties analyze(ConfigListener conf) throws ConfigException {
    final Map<String, Object> newProperties = new HashMap<String, Object>(conf.properties.size() + Properties.PROPSNAMES.values().length + System.getProperties().size());
    // Resolvers properties found and and it to new properties
    Function<Object, Object> resolve = i -> {
        return ((i instanceof ConfigListener.ObjectWrapped) ? ((ConfigListener.ObjectWrapped) i).wrapped : (i instanceof ConfigListener.ObjectReference) ? parseObjectDescription((ConfigListener.ObjectDescription) i, emptyConstructor) : i);
    };
    conf.properties.entrySet().stream().forEach(i -> newProperties.put(i.getKey(), resolve.apply(i.getValue())));
    // Resolve the sources
    ThrowingFunction<Class<Source>, Source> sourceConstructor = i -> {
        return i.getConstructor().newInstance();
    };
    conf.sources.forEach((name, sd) -> {
        Source s = parseObjectDescription(sd, sourceConstructor);
        s.setName(name);
        sources.put(name, s);
    });
    Map<String, Pipeline> namedPipeLine = new HashMap<>(conf.pipelines.size());
    newProperties.put(Properties.PROPSNAMES.CLASSLOADERNAME.toString(), classLoader);
    Set<Pipeline> pipelines = new HashSet<>();
    // Generate all the named pipeline
    for (Entry<String, ConfigListener.PipenodesList> e : conf.pipelines.entrySet()) {
        String name = e.getKey();
        Pipeline p = parsePipeline(e.getValue(), name, 0, new AtomicInteger());
        pipelines.add(p);
        namedPipeLine.put(name, p);
        if (p.nextPipeline != null) {
            topPipelines.add(p.nextPipeline);
        }
    }
    newProperties.put(Properties.PROPSNAMES.PIPELINES.toString(), Collections.unmodifiableSet(pipelines));
    namedPipeLine = Collections.unmodifiableMap(namedPipeLine);
    newProperties.put(Properties.PROPSNAMES.NAMEDPIPELINES.toString(), namedPipeLine);
    // Find the queue depth
    final int queuesDepth = newProperties.containsKey("queueDepth") ? (Integer) newProperties.remove("queueDepth") : DEFAULTQUEUEDEPTH;
    newProperties.put(Properties.PROPSNAMES.QUEUESDEPTH.toString(), queuesDepth);
    BlockingQueue<Event> mainQueue = new ArrayBlockingQueue<Event>(queuesDepth);
    Map<String, BlockingQueue<Event>> outputQueues = new HashMap<>(namedPipeLine.size());
    namedPipeLine.keySet().stream().forEach(i -> outputQueues.put(i, new ArrayBlockingQueue<Event>(queuesDepth)));
    newProperties.put(Properties.PROPSNAMES.FORMATTERS.toString(), conf.formatters);
    newProperties.put(Properties.PROPSNAMES.MAINQUEUE.toString(), mainQueue);
    newProperties.put(Properties.PROPSNAMES.OUTPUTQUEUE.toString(), outputQueues);
    // Fill the receivers list
    receivers = new ArrayList<>();
    for (Input i : conf.inputs) {
        if (i.piperef == null || !namedPipeLine.containsKey(i.piperef)) {
            throw new RuntimeException("Invalid input, no destination pipeline: " + i);
        }
        for (ConfigListener.ObjectDescription desc : i.receiver) {
            Pipeline p = namedPipeLine.get(i.piperef);
            ThrowingFunction<Class<Receiver>, Receiver> receiverConstructor = r -> {
                return r.getConstructor(BlockingQueue.class, Pipeline.class).newInstance(mainQueue, p);
            };
            Receiver r = (Receiver) parseObjectDescription(desc, receiverConstructor);
            receivers.add(r);
        }
        inputpipelines.add(i.piperef);
    }
    topPipelines.addAll(inputpipelines);
    inputpipelines = Collections.unmodifiableSet(inputpipelines);
    receivers = Collections.unmodifiableList(receivers);
    newProperties.put(Properties.PROPSNAMES.RECEIVERS.toString(), receivers);
    // Fill the senders list
    senders = new ArrayList<>();
    for (Output o : conf.outputs) {
        if (o.piperef == null || !namedPipeLine.containsKey(o.piperef)) {
            throw new RuntimeException("Invalid output, no source pipeline: " + o);
        }
        for (ConfigListener.ObjectDescription desc : o.sender) {
            BlockingQueue<Event> out = outputQueues.get(o.piperef);
            ThrowingFunction<Class<Sender>, Sender> senderConstructor = r -> {
                return r.getConstructor(BlockingQueue.class).newInstance(out);
            };
            Sender s = (Sender) parseObjectDescription(desc, senderConstructor);
            // logger.debug("sender {} source point will be {}", () -> s, () -> namedPipeLine.get(o.piperef).outQueue);
            senders.add(s);
        }
        outputpipelines.add(o.piperef);
    }
    topPipelines.addAll(outputpipelines);
    outputpipelines = Collections.unmodifiableSet(outputpipelines);
    senders = Collections.unmodifiableList(senders);
    newProperties.put(Properties.PROPSNAMES.SENDERS.toString(), senders);
    newProperties.put(Properties.PROPSNAMES.TOPPIPELINE.toString(), Collections.unmodifiableSet(topPipelines));
    newProperties.put(Properties.PROPSNAMES.SOURCES.toString(), Collections.unmodifiableMap(sources));
    // Allows the system properties to override any properties given in the configuration file
    // But only if they are not some of the special internal properties
    Set<String> privatepropsnames = new HashSet<>(Properties.PROPSNAMES.values().length);
    Arrays.stream(Properties.PROPSNAMES.values()).forEach(i -> privatepropsnames.add(i.toString()));
    ;
    System.getProperties().entrySet().stream().filter(i -> !privatepropsnames.contains(i.getKey())).forEach(i -> newProperties.put(i.getKey().toString(), i.getValue()));
    return new Properties(newProperties);
}

Example 18 with Event

use of loghub.Event in project LogHub by fbacchella.

the class TestConfigurations method testBuildSubPipeline.

@Test
public void testBuildSubPipeline() throws IOException, InterruptedException, ConfigException {
    Properties conf = Tools.loadConf("simple.conf");
    Event sent = Tools.getEvent();
    logger.debug("pipelines: " + conf.pipelines);
    logger.debug("namedPipeLine: " + conf.namedPipeLine);
    conf.mainQueue.add(sent);
    Event received = conf.mainQueue.poll(1, TimeUnit.SECONDS);
    Assert.assertEquals("not expected event received", sent, received);
}

Example 19 with Event

use of loghub.Event in project LogHub by fbacchella.

the class TestConfigurations method testFork.

@Test
public void testFork() throws InterruptedException, ProcessorException, ConfigException, IOException {
    Properties conf = Tools.loadConf("forkforward.conf");
    EventsProcessor ep = new EventsProcessor(conf.mainQueue, conf.outputQueues, conf.namedPipeLine, conf.maxSteps, conf.repository);
    ep.start();
    try {
        Event sent = Tools.getEvent();
        sent.inject(conf.namedPipeLine.get("mainfork"), conf.mainQueue);
        Event forked = conf.outputQueues.get("forked").poll(1, TimeUnit.SECONDS);
        Event initial = conf.outputQueues.get("mainfork").poll(1, TimeUnit.SECONDS);
        Assert.assertEquals(1, forked.size());
        Assert.assertEquals(1, initial.size());
        Assert.assertEquals(2, forked.get("b"));
        Assert.assertEquals(1, initial.get("a"));
    } finally {
        ep.interrupt();
    }
}

Example 20 with Event

use of loghub.Event in project LogHub by fbacchella.

the class TestConfigurations method testForward.

@Test
public void testForward() throws InterruptedException, ProcessorException, ConfigException, IOException {
    Properties conf = Tools.loadConf("forkforward.conf");
    EventsProcessor ep = new EventsProcessor(conf.mainQueue, conf.outputQueues, conf.namedPipeLine, conf.maxSteps, conf.repository);
    ep.start();
    try {
        Event sent = Tools.getEvent();
        sent.inject(conf.namedPipeLine.get("mainforward"), conf.mainQueue);
        Event forwarded = conf.outputQueues.get("forked").poll(1, TimeUnit.SECONDS);
        Event initial = conf.outputQueues.get("mainforward").poll(1, TimeUnit.SECONDS);
        Assert.assertEquals(1, forwarded.size());
        Assert.assertNull(initial);
        Assert.assertEquals(2, forwarded.get("b"));
        Assert.assertNull(sent.get("a"));
    } finally {
        ep.interrupt();
    }
}