Examples with EventDeliveryException org.apache.flume.EventDeliveryException used on opensource projects

Example 1 with EventDeliveryException

use of org.apache.flume.EventDeliveryException in project phoenix by apache.

the class PhoenixSink method process.

@Override
public Status process() throws EventDeliveryException {
    Status status = Status.READY;
    Channel channel = getChannel();
    Transaction transaction = null;
    List<Event> events = Lists.newArrayListWithExpectedSize(this.batchSize);
    long startTime = System.nanoTime();
    try {
        transaction = channel.getTransaction();
        transaction.begin();
        for (long i = 0; i < this.batchSize; i++) {
            Event event = channel.take();
            if (event == null) {
                status = Status.BACKOFF;
                if (i == 0) {
                    sinkCounter.incrementBatchEmptyCount();
                } else {
                    sinkCounter.incrementBatchUnderflowCount();
                }
                break;
            } else {
                events.add(event);
            }
        }
        if (!events.isEmpty()) {
            if (events.size() == this.batchSize) {
                sinkCounter.incrementBatchCompleteCount();
            } else {
                sinkCounter.incrementBatchUnderflowCount();
                status = Status.BACKOFF;
            }
            // save to Hbase
            serializer.upsertEvents(events);
            sinkCounter.addToEventDrainSuccessCount(events.size());
        } else {
            logger.debug("no events to process ");
            sinkCounter.incrementBatchEmptyCount();
            status = Status.BACKOFF;
        }
        transaction.commit();
    } catch (ChannelException e) {
        transaction.rollback();
        status = Status.BACKOFF;
        sinkCounter.incrementConnectionFailedCount();
    } catch (SQLException e) {
        sinkCounter.incrementConnectionFailedCount();
        transaction.rollback();
        logger.error("exception while persisting to Hbase ", e);
        throw new EventDeliveryException("Failed to persist message to Hbase", e);
    } catch (Throwable e) {
        transaction.rollback();
        logger.error("exception while processing in Phoenix Sink", e);
        throw new EventDeliveryException("Failed to persist message", e);
    } finally {
        logger.info(String.format("Time taken to process [%s] events was [%s] seconds", events.size(), TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS)));
        if (transaction != null) {
            transaction.close();
        }
    }
    return status;
}

Example 2 with EventDeliveryException

use of org.apache.flume.EventDeliveryException in project nifi by apache.

the class ExecuteFlumeSource method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    if (source instanceof PollableSource) {
        PollableSource pollableSource = (PollableSource) source;
        try {
            pollableSourceChannel.setSession(session);
            pollableSource.process();
        } catch (EventDeliveryException ex) {
            throw new ProcessException("Error processing pollable source", ex);
        }
    } else {
        throw new ProcessException("Invalid source type: " + source.getClass().getSimpleName());
    }
}

Example 3 with EventDeliveryException

use of org.apache.flume.EventDeliveryException in project rocketmq-externals by apache.

the class RocketMQSource method doProcess.

@Override
protected Status doProcess() throws EventDeliveryException {
    List<Event> events = new ArrayList<>();
    Map<MessageQueue, Long> offsets = new HashMap<>();
    Event event;
    Map<String, String> headers;
    try {
        Set<MessageQueue> queues = consumer.fetchSubscribeMessageQueues(topic);
        for (MessageQueue queue : queues) {
            long offset = getMessageQueueOffset(queue);
            PullResult pullResult = consumer.pull(queue, tag, offset, batchSize);
            if (log.isDebugEnabled()) {
                log.debug("Pull from queueId:{}, offset:{}, pullResult:{}", queue.getQueueId(), offset, pullResult);
            }
            if (pullResult.getPullStatus() == PullStatus.FOUND) {
                for (MessageExt msg : pullResult.getMsgFoundList()) {
                    byte[] body = msg.getBody();
                    headers = new HashMap<>();
                    headers.put(HEADER_TOPIC_NAME, topic);
                    headers.put(HEADER_TAG_NAME, tag);
                    if (log.isDebugEnabled()) {
                        log.debug("Processing message,body={}", new String(body, "UTF-8"));
                    }
                    event = EventBuilder.withBody(body, headers);
                    events.add(event);
                }
                offsets.put(queue, pullResult.getNextBeginOffset());
            }
        }
        if (events.size() > 0) {
            sourceCounter.incrementAppendBatchReceivedCount();
            sourceCounter.addToEventReceivedCount(events.size());
            getChannelProcessor().processEventBatch(events);
            sourceCounter.incrementAppendBatchAcceptedCount();
            sourceCounter.addToEventAcceptedCount(events.size());
            events.clear();
        }
        for (Map.Entry<MessageQueue, Long> entry : offsets.entrySet()) {
            putMessageQueueOffset(entry.getKey(), entry.getValue());
        }
    } catch (Exception e) {
        log.error("Failed to consumer message", e);
        return Status.BACKOFF;
    }
    return Status.READY;
}

Example 4 with EventDeliveryException

use of org.apache.flume.EventDeliveryException in project cdap-ingest by caskdata.

the class StreamSink method process.

@Override
public Status process() throws EventDeliveryException {
    Status status = Status.READY;
    Channel channel = getChannel();
    Transaction transaction = channel.getTransaction();
    try {
        tryReopenClientConnection();
        transaction.begin();
        Event event = channel.take();
        if (event != null) {
            try {
                writer.write(ByteBuffer.wrap(event.getBody()), event.getHeaders()).get();
                LOG.trace("Success write to stream: {} ", streamName);
            } catch (Throwable t) {
                if (t instanceof ExecutionException) {
                    t = t.getCause();
                }
                LOG.error("Error during writing event to stream {}", streamName, t);
                throw new EventDeliveryException("Failed to send events to stream: " + streamName, t);
            }
        }
        transaction.commit();
    } catch (Throwable t) {
        transaction.rollback();
        if (t instanceof Error) {
            throw (Error) t;
        } else if (t instanceof ChannelException) {
            LOG.error("Stream Sink {}: Unable to get event from channel {} ", getName(), channel.getName());
            status = Status.BACKOFF;
        } else {
            LOG.debug("Closing writer due to stream error ", t);
            closeClientQuietly();
            closeWriterQuietly();
            throw new EventDeliveryException("Sink event sending error", t);
        }
    } finally {
        transaction.close();
    }
    return status;
}

Example 5 with EventDeliveryException

use of org.apache.flume.EventDeliveryException in project phoenix by apache.

the class CsvEventSerializerIT method testMismatchKeyGenerator.

@Test
public void testMismatchKeyGenerator() throws EventDeliveryException, SQLException {
    final String fullTableName = "FLUME_CSV_TEST";
    initSinkContextWithDefaults(fullTableName);
    setConfig(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR, DefaultKeyGenerator.UUID.name());
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    sink.start();
    final String eventBody = "kalyan,10.5,\"abc,pqr,xyz\",\"1,2,3,4\"";
    final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event);
    transaction.commit();
    transaction.close();
    try {
        sink.process();
        fail();
    } catch (Exception ex) {
        assertTrue(ex.getCause().getMessage().contains("java.lang.IllegalArgumentException: Invalid format:"));
    }
    dropTable(fullTableName);
}