Examples with Checkpoint io.cdap.cdap.logging.meta.Checkpoint used on opensource projects

Example 6 with Checkpoint

use of io.cdap.cdap.logging.meta.Checkpoint in project cdap by cdapio.

the class TimeEventQueueProcessorTest method test.

@Test
public void test() throws Exception {
    LoggerContext loggerContext = LogPipelineTestUtil.createLoggerContext("WARN", ImmutableMap.of("test.logger", "INFO"), MockAppender.class.getName());
    LogProcessorPipelineContext context = new LogProcessorPipelineContext(CConfiguration.create(), "test", loggerContext, NO_OP_METRICS_CONTEXT, 0);
    context.start();
    TimeEventQueueProcessor<TestOffset> processor = new TimeEventQueueProcessor<>(context, 50, 1, ImmutableList.of(0));
    long now = System.currentTimeMillis();
    List<ILoggingEvent> events = ImmutableList.of(LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "1", now - 1000), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "3", now - 700), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "5", now - 500), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "2", now - 900), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.ERROR, "4", now - 600), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "6", now - 100));
    ProcessedEventMetadata<TestOffset> metadata = processor.process(0, new TransformingIterator(events.iterator()));
    // all 6 events should be processed. This is because when the buffer is full after 5 events, time event queue
    // processor should append existing buffered events and enqueue 6th event
    Assert.assertEquals(6, metadata.getTotalEventsProcessed());
    for (Map.Entry<Integer, Checkpoint<TestOffset>> entry : metadata.getCheckpoints().entrySet()) {
        Checkpoint<TestOffset> value = entry.getValue();
        // offset should be max offset processed so far
        Assert.assertEquals(6, value.getOffset().getOffset());
    }
}

Example 7 with Checkpoint

use of io.cdap.cdap.logging.meta.Checkpoint in project cdap by cdapio.

the class TestDistributedLogReader method generateCheckpointTime.

private static void generateCheckpointTime(LoggingContext loggingContext, int numExpectedEvents, String kafkaTopic) throws IOException {
    FileLogReader logReader = injector.getInstance(FileLogReader.class);
    List<LogEvent> events = Lists.newArrayList(logReader.getLog(loggingContext, 0, Long.MAX_VALUE, Filter.EMPTY_FILTER));
    Assert.assertEquals(numExpectedEvents, events.size());
    // Save checkpoint (time of last event)
    TransactionRunner transactionRunner = injector.getInstance(TransactionRunner.class);
    CheckpointManager<KafkaOffset> checkpointManager = new KafkaCheckpointManager(transactionRunner, Constants.Logging.SYSTEM_PIPELINE_CHECKPOINT_PREFIX + kafkaTopic);
    long checkpointTime = events.get(numExpectedEvents - 1).getLoggingEvent().getTimeStamp();
    checkpointManager.saveCheckpoints(ImmutableMap.of(stringPartitioner.partition(loggingContext.getLogPartition(), -1), new Checkpoint<>(new KafkaOffset(numExpectedEvents, checkpointTime), checkpointTime)));
}

Example 8 with Checkpoint

use of io.cdap.cdap.logging.meta.Checkpoint in project cdap by caskdata.

the class TestDistributedLogReader method generateCheckpointTime.

private static void generateCheckpointTime(LoggingContext loggingContext, int numExpectedEvents, String kafkaTopic) throws IOException {
    FileLogReader logReader = injector.getInstance(FileLogReader.class);
    List<LogEvent> events = Lists.newArrayList(logReader.getLog(loggingContext, 0, Long.MAX_VALUE, Filter.EMPTY_FILTER));
    Assert.assertEquals(numExpectedEvents, events.size());
    // Save checkpoint (time of last event)
    TransactionRunner transactionRunner = injector.getInstance(TransactionRunner.class);
    CheckpointManager<KafkaOffset> checkpointManager = new KafkaCheckpointManager(transactionRunner, Constants.Logging.SYSTEM_PIPELINE_CHECKPOINT_PREFIX + kafkaTopic);
    long checkpointTime = events.get(numExpectedEvents - 1).getLoggingEvent().getTimeStamp();
    checkpointManager.saveCheckpoints(ImmutableMap.of(stringPartitioner.partition(loggingContext.getLogPartition(), -1), new Checkpoint<>(new KafkaOffset(numExpectedEvents, checkpointTime), checkpointTime)));
}

Example 9 with Checkpoint

use of io.cdap.cdap.logging.meta.Checkpoint in project cdap by caskdata.

the class DistributedLogFrameworkTest method testFramework.

@Test
public void testFramework() throws Exception {
    DistributedLogFramework framework = injector.getInstance(DistributedLogFramework.class);
    CConfiguration cConf = injector.getInstance(CConfiguration.class);
    framework.startAndWait();
    // Send some logs to Kafka.
    LoggingContext context = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, "test");
    // Make sure all events get flushed in the same batch
    long eventTimeBase = System.currentTimeMillis() + cConf.getInt(Constants.Logging.PIPELINE_EVENT_DELAY_MS);
    final int msgCount = 50;
    for (int i = 0; i < msgCount; i++) {
        // Publish logs in descending timestamp order
        publishLog(cConf.get(Constants.Logging.KAFKA_TOPIC), context, ImmutableList.of(createLoggingEvent("io.cdap.test." + i, Level.INFO, "Testing " + i, eventTimeBase - i)));
    }
    // Read the logs back. They should be sorted by timestamp order.
    final FileMetaDataReader metaDataReader = injector.getInstance(FileMetaDataReader.class);
    Tasks.waitFor(true, () -> {
        List<LogLocation> locations = metaDataReader.listFiles(new LogPathIdentifier(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, "test"), 0, Long.MAX_VALUE);
        if (locations.size() != 1) {
            return false;
        }
        LogLocation location = locations.get(0);
        int i = 0;
        try {
            try (CloseableIterator<LogEvent> iter = location.readLog(Filter.EMPTY_FILTER, 0, Long.MAX_VALUE, msgCount)) {
                while (iter.hasNext()) {
                    String expectedMsg = "Testing " + (msgCount - i - 1);
                    LogEvent event = iter.next();
                    if (!expectedMsg.equals(event.getLoggingEvent().getMessage())) {
                        return false;
                    }
                    i++;
                }
                return i == msgCount;
            }
        } catch (Exception e) {
            // and the time when actual content are flushed to the file
            return false;
        }
    }, 10, TimeUnit.SECONDS, msgCount, TimeUnit.MILLISECONDS);
    framework.stopAndWait();
    String kafkaTopic = cConf.get(Constants.Logging.KAFKA_TOPIC);
    // Check the checkpoint is persisted correctly. Since all messages are processed,
    // the checkpoint should be the same as the message count.
    CheckpointManager<KafkaOffset> checkpointManager = getCheckpointManager(kafkaTopic);
    Checkpoint<KafkaOffset> checkpoint = checkpointManager.getCheckpoint(0);
    Assert.assertEquals(msgCount, checkpoint.getOffset().getNextOffset());
}

Example 10 with Checkpoint

use of io.cdap.cdap.logging.meta.Checkpoint in project cdap by caskdata.

the class KafkaLogProcessorPipeline method initializeOffsets.

/**
 * Initialize offsets for all partitions consumed by this pipeline.
 *
 * @throws InterruptedException if there is an interruption
 */
private void initializeOffsets() throws InterruptedException {
    // Setup initial offsets
    Set<Integer> partitions = new HashSet<>(config.getPartitions());
    while (!partitions.isEmpty() && !stopped) {
        Iterator<Integer> iterator = partitions.iterator();
        boolean failed = false;
        while (iterator.hasNext()) {
            int partition = iterator.next();
            MutableCheckpoint checkpoint = checkpoints.get(partition);
            try {
                if (checkpoint == null || checkpoint.getOffset().nextOffset <= 0) {
                    // If no checkpoint, fetch from the beginning.
                    offsets.put(partition, getLastOffset(partition, kafka.api.OffsetRequest.EarliestTime()));
                } else {
                    // Otherwise, validate and find the offset if not valid
                    MutableKafkaOffset kafkaOffset = checkpoint.getOffset();
                    offsets.put(partition, offsetResolver.getStartOffset(kafkaOffset.nextOffset, kafkaOffset.nextEventTime, partition));
                }
                // Remove the partition successfully stored in offsets to avoid unnecessary retry for this partition
                iterator.remove();
            } catch (Exception e) {
                OUTAGE_LOG.warn("Failed to get a valid offset from Kafka to start consumption for {}:{}", config.getTopic(), partition);
                failed = true;
            }
        }
        // Should keep finding valid offsets for all partitions
        if (failed && !stopped) {
            TimeUnit.SECONDS.sleep(2L);
        }
    }
}