Examples with KeyValueSink com.mozilla.bagheera.sink.KeyValueSink used on opensource projects

Example 1 with KeyValueSink

use of com.mozilla.bagheera.sink.KeyValueSink in project bagheera by mozilla-metrics.

the class KafkaConsumer method poll.

@Override
public void poll() {
    final CountDownLatch latch = new CountDownLatch(streams.size());
    for (final KafkaStream<Message> stream : streams) {
        workers.add(executor.submit(new Callable<Void>() {

            @Override
            public Void call() {
                try {
                    for (MessageAndMetadata<Message> mam : stream) {
                        BagheeraMessage bmsg = BagheeraMessage.parseFrom(ByteString.copyFrom(mam.message().payload()));
                        // get the sink for this message's namespace 
                        // (typically only one sink unless a regex pattern was used to listen to multiple topics)
                        KeyValueSink sink = sinkFactory.getSink(bmsg.getNamespace());
                        if (sink == null) {
                            LOG.error("Could not obtain sink for namespace: " + bmsg.getNamespace());
                            break;
                        }
                        if (bmsg.getOperation() == Operation.CREATE_UPDATE && bmsg.hasId() && bmsg.hasPayload()) {
                            if (validationPipeline == null || validationPipeline.isValid(bmsg.getPayload().toByteArray())) {
                                if (bmsg.hasTimestamp()) {
                                    sink.store(bmsg.getId(), bmsg.getPayload().toByteArray(), bmsg.getTimestamp());
                                } else {
                                    sink.store(bmsg.getId(), bmsg.getPayload().toByteArray());
                                }
                            } else {
                                invalidMessageMeter.mark();
                                // TODO: sample out an example payload
                                LOG.warn("Invalid payload for namespace: " + bmsg.getNamespace());
                            }
                        } else if (bmsg.getOperation() == Operation.DELETE && bmsg.hasId()) {
                            sink.delete(bmsg.getId());
                        }
                        consumed.mark();
                    }
                } catch (InvalidProtocolBufferException e) {
                    LOG.error("Invalid protocol buffer in data stream", e);
                } catch (UnsupportedEncodingException e) {
                    LOG.error("Message ID was not in UTF-8 encoding", e);
                } catch (IOException e) {
                    LOG.error("IO error while storing to data sink", e);
                } finally {
                    latch.countDown();
                }
                return null;
            }
        }));
    }
    // run indefinitely unless we detect that a thread exited
    try {
        while (true) {
            latch.await(10, TimeUnit.SECONDS);
            if (latch.getCount() != streams.size()) {
                // we have a dead thread and should exit
                break;
            }
        }
    } catch (InterruptedException e) {
        LOG.info("Interrupted during polling", e);
    }
    // Spit out errors if there were any
    for (Future<Void> worker : workers) {
        try {
            if (worker.isDone() && !worker.isCancelled()) {
                worker.get(1, TimeUnit.SECONDS);
            }
        } catch (InterruptedException e) {
            LOG.error("Thread was interrupted:", e);
        } catch (ExecutionException e) {
            LOG.error("Exception occured in thread:", e);
        } catch (TimeoutException e) {
            LOG.error("Timed out waiting for thread result:", e);
        } catch (CancellationException e) {
            LOG.error("Thread has been canceled: ", e);
        }
    }
}