Examples with Checkpoint io.cdap.cdap.logging.meta.Checkpoint used on opensource projects

Example 1 with Checkpoint

use of io.cdap.cdap.logging.meta.Checkpoint in project cdap by caskdata.

the class KafkaLogProcessorPipeline method startUp.

@Override
protected void startUp() throws Exception {
    LOG.debug("Starting log processor pipeline for {} with configurations {}", name, config);
    // Reads the existing checkpoints
    Set<Integer> partitions = config.getPartitions();
    for (Map.Entry<Integer, Checkpoint<KafkaOffset>> entry : checkpointManager.getCheckpoint(partitions).entrySet()) {
        Checkpoint<KafkaOffset> checkpoint = entry.getValue();
        KafkaOffset kafkaOffset = checkpoint.getOffset();
        // Skip the partition that doesn't have previous checkpoint.
        if (kafkaOffset.getNextOffset() >= 0 && kafkaOffset.getNextEventTime() >= 0 && checkpoint.getMaxEventTime() >= 0) {
            checkpoints.put(entry.getKey(), new MutableCheckpoint(checkpoint));
        }
    }
    context.start();
    fetchExecutor = Executors.newFixedThreadPool(partitions.size(), Threads.createDaemonThreadFactory("fetcher-" + name + "-%d"));
    // emit pipeline related config as metrics
    emitConfigMetrics();
    LOG.info("Log processor pipeline for {} with config {} started with checkpoint {}", name, config, checkpoints);
}

Example 2 with Checkpoint

use of io.cdap.cdap.logging.meta.Checkpoint in project cdap by caskdata.

the class KafkaLogProcessorPipeline method processMessages.

/**
 * Processes log messages for given partition. This method will provide events iterator to event queue processor
 * and update checkpoints and offsets based on events processed by event queue processor. If any of the events are
 * processed by event queue processor, method returns true, otherwise false.
 */
private boolean processMessages(String topic, int partition, Future<Iterable<MessageAndOffset>> future) throws InterruptedException, KafkaException, IOException {
    Iterable<MessageAndOffset> messages;
    try {
        messages = future.get();
    } catch (ExecutionException e) {
        try {
            throw e.getCause();
        } catch (OffsetOutOfRangeException cause) {
            // This shouldn't happen under normal situation.
            // If happened, usually is caused by race between kafka log rotation and fetching in here,
            // hence just fetching from the beginning should be fine
            offsets.put(partition, getLastOffset(partition, kafka.api.OffsetRequest.EarliestTime()));
            return false;
        } catch (KafkaException | IOException cause) {
            throw cause;
        } catch (Throwable t) {
            // For other type of exceptions, just throw an IOException. It will be handled by caller.
            throw new IOException(t);
        }
    }
    // process all the messages
    ProcessedEventMetadata<KafkaOffset> metadata = eventQueueProcessor.process(partition, new KafkaMessageTransformIterator(topic, partition, messages.iterator()));
    // None of the events were processed.
    if (metadata.getTotalEventsProcessed() <= 0) {
        return false;
    }
    // only update checkpoints if some events were processed
    unSyncedEvents += metadata.getTotalEventsProcessed();
    for (Map.Entry<Integer, Checkpoint<KafkaOffset>> entry : metadata.getCheckpoints().entrySet()) {
        MutableCheckpoint checkpoint = checkpoints.get(entry.getKey());
        Checkpoint<KafkaOffset> checkpointMetadata = entry.getValue();
        // Update checkpoints
        KafkaOffset kafkaOffset = checkpointMetadata.getOffset();
        if (checkpoint == null) {
            checkpoint = new MutableCheckpoint(kafkaOffset, checkpointMetadata.getMaxEventTime());
            checkpoints.put(entry.getKey(), checkpoint);
        } else {
            MutableKafkaOffset offset = checkpoint.getOffset();
            offset.setNextOffset(kafkaOffset.getNextOffset());
            offset.setNextEventTime(kafkaOffset.getNextEventTime());
            checkpoint.setMaxEventTime(checkpointMetadata.getMaxEventTime());
        }
    }
    // For each partition, if there is no more event in the event queue, update the checkpoint nextOffset
    for (Int2LongMap.Entry entry : offsets.int2LongEntrySet()) {
        if (eventQueueProcessor.isQueueEmpty(entry.getIntKey())) {
            MutableCheckpoint checkpoint = checkpoints.get(entry.getIntKey());
            long offset = entry.getLongValue();
            // it means everything before the process offset must had been written to the appender.
            if (checkpoint != null && offset > checkpoint.getOffset().getNextOffset()) {
                checkpoint.getOffset().setNextOffset(offset);
            }
        }
    }
    return true;
}

Example 3 with Checkpoint

use of io.cdap.cdap.logging.meta.Checkpoint in project cdap by caskdata.

the class TimeEventQueueProcessor method process.

/**
 * Processes events provided by event iterator for a given partition.
 *
 * @param partition log event partition
 * @param eventIterator processor events iterator
 *
 * @return processed event metadata
 */
public ProcessedEventMetadata<OFFSET> process(int partition, Iterator<ProcessorEvent<OFFSET>> eventIterator) {
    int totalEvents = 0;
    Map<Integer, Checkpoint<OFFSET>> checkpoints = new HashMap<>();
    // iterate through all the events if buffer size has not reached max
    while (eventIterator.hasNext()) {
        if (eventQueue.getEventSize() >= maxBufferSize) {
            OUTAGE_LOG.info("Maximum queue size {} reached for log pipeline {}.", maxBufferSize, context.getName());
            // Event queue is full. So try to append events to log appenders. If none of the events are appended to log
            // appenders, then do not enqueue any more events.
            ProcessedEventMetadata<OFFSET> eventsMetadata = append();
            if (eventsMetadata.getTotalEventsProcessed() <= 0) {
                break;
            }
            totalEvents += eventsMetadata.getTotalEventsProcessed();
            checkpoints.putAll(eventsMetadata.getCheckpoints());
        }
        ProcessorEvent<OFFSET> processorEvent = eventIterator.next();
        eventQueue.add(processorEvent.getEvent(), processorEvent.getEvent().getTimeStamp(), processorEvent.getEventSize(), partition, processorEvent.getOffset());
    }
    // if event queue is full or all the events have been added to the queue, append all the enqueued events to log
    // appenders.
    ProcessedEventMetadata<OFFSET> eventsMetadata = append();
    if (eventsMetadata.getTotalEventsProcessed() > 0) {
        totalEvents += eventsMetadata.getTotalEventsProcessed();
        checkpoints.putAll(eventsMetadata.getCheckpoints());
    }
    return new ProcessedEventMetadata<>(totalEvents, checkpoints);
}

Example 4 with Checkpoint

use of io.cdap.cdap.logging.meta.Checkpoint in project cdap by caskdata.

the class TimeEventQueueProcessorTest method test.

@Test
public void test() throws Exception {
    LoggerContext loggerContext = LogPipelineTestUtil.createLoggerContext("WARN", ImmutableMap.of("test.logger", "INFO"), MockAppender.class.getName());
    LogProcessorPipelineContext context = new LogProcessorPipelineContext(CConfiguration.create(), "test", loggerContext, NO_OP_METRICS_CONTEXT, 0);
    context.start();
    TimeEventQueueProcessor<TestOffset> processor = new TimeEventQueueProcessor<>(context, 50, 1, ImmutableList.of(0));
    long now = System.currentTimeMillis();
    List<ILoggingEvent> events = ImmutableList.of(LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "1", now - 1000), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "3", now - 700), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "5", now - 500), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "2", now - 900), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.ERROR, "4", now - 600), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "6", now - 100));
    ProcessedEventMetadata<TestOffset> metadata = processor.process(0, new TransformingIterator(events.iterator()));
    // all 6 events should be processed. This is because when the buffer is full after 5 events, time event queue
    // processor should append existing buffered events and enqueue 6th event
    Assert.assertEquals(6, metadata.getTotalEventsProcessed());
    for (Map.Entry<Integer, Checkpoint<TestOffset>> entry : metadata.getCheckpoints().entrySet()) {
        Checkpoint<TestOffset> value = entry.getValue();
        // offset should be max offset processed so far
        Assert.assertEquals(6, value.getOffset().getOffset());
    }
}

Example 5 with Checkpoint

use of io.cdap.cdap.logging.meta.Checkpoint in project cdap by caskdata.

the class TestDistributedLogReader method generateCheckpointTime.

private static void generateCheckpointTime(LoggingContext loggingContext, int numExpectedEvents, String kafkaTopic) throws IOException {
    FileLogReader logReader = injector.getInstance(FileLogReader.class);
    List<LogEvent> events = Lists.newArrayList(logReader.getLog(loggingContext, 0, Long.MAX_VALUE, Filter.EMPTY_FILTER));
    Assert.assertEquals(numExpectedEvents, events.size());
    // Save checkpoint (time of last event)
    TransactionRunner transactionRunner = injector.getInstance(TransactionRunner.class);
    CheckpointManager<KafkaOffset> checkpointManager = new KafkaCheckpointManager(transactionRunner, Constants.Logging.SYSTEM_PIPELINE_CHECKPOINT_PREFIX + kafkaTopic);
    long checkpointTime = events.get(numExpectedEvents - 1).getLoggingEvent().getTimeStamp();
    checkpointManager.saveCheckpoints(ImmutableMap.of(stringPartitioner.partition(loggingContext.getLogPartition(), -1), new Checkpoint<>(new KafkaOffset(numExpectedEvents, checkpointTime), checkpointTime)));
}