Examples with ReadLogEntry com.alibaba.graphscope.groot.wal.ReadLogEntry used on opensource projects

Example 1 with ReadLogEntry

use of com.alibaba.graphscope.groot.wal.ReadLogEntry in project GraphScope by alibaba.

the class KafkaWalTest method testLogService.

@Test
void testLogService() throws IOException {
    Configs configs = Configs.newBuilder().put(KafkaConfig.KAFKA_SERVERS.getKey(), sharedKafkaTestResource.getKafkaConnectString()).put(KafkaConfig.KAKFA_TOPIC.getKey(), "test_logservice").put(CommonConfig.INGESTOR_QUEUE_COUNT.getKey(), "1").build();
    LogService logService = new KafkaLogService(configs);
    logService.init();
    int queueId = 0;
    long snapshotId = 1L;
    LogWriter writer = logService.createWriter(queueId);
    LogEntry logEntry = new LogEntry(snapshotId, OperationBatch.newBuilder().addOperationBlob(OperationBlob.MARKER_OPERATION_BLOB).build());
    assertEquals(writer.append(logEntry), 0);
    LogReader reader = logService.createReader(queueId, 0);
    ReadLogEntry readLogEntry = reader.readNext();
    reader.close();
    assertAll(() -> assertEquals(readLogEntry.getOffset(), 0), () -> assertEquals(readLogEntry.getLogEntry().getSnapshotId(), snapshotId));
    OperationBatch operationBatch = readLogEntry.getLogEntry().getOperationBatch();
    assertEquals(operationBatch.getOperationCount(), 1);
    assertEquals(operationBatch.getOperationBlob(0), OperationBlob.MARKER_OPERATION_BLOB);
    assertEquals(writer.append(logEntry), 1);
    assertEquals(writer.append(logEntry), 2);
    assertEquals(writer.append(logEntry), 3);
    LogReader readerTail = logService.createReader(queueId, 4);
    assertNull(readerTail.readNext());
    readerTail.close();
    assertThrows(IllegalArgumentException.class, () -> logService.createReader(queueId, 5));
    logService.deleteBeforeOffset(queueId, 2);
    assertThrows(IllegalArgumentException.class, () -> logService.createReader(queueId, 1));
    writer.close();
    logService.destroy();
}

Example 2 with ReadLogEntry

use of com.alibaba.graphscope.groot.wal.ReadLogEntry in project GraphScope by alibaba.

the class IngestProcessorTest method testIngestProcessor.

@Test
void testIngestProcessor() throws IOException {
    long tailOffset = 50L;
    int queueId = 0;
    Configs configs = Configs.newBuilder().build();
    LogService mockLogService = mock(LogService.class);
    LogReader mockLogReader = mock(LogReader.class);
    when(mockLogService.createReader(queueId, tailOffset + 1)).thenReturn(mockLogReader);
    LogWriter mockLogWriter = mock(LogWriter.class);
    when(mockLogService.createWriter(queueId)).thenReturn(mockLogWriter);
    when(mockLogWriter.append(any())).thenReturn(tailOffset + 3);
    OperationBatch emptyBatch = OperationBatch.newBuilder().build();
    long readSnapshotId = 5L;
    ReadLogEntry readLogEntry1 = new ReadLogEntry(tailOffset + 1, readSnapshotId, emptyBatch);
    ReadLogEntry readLogEntry2 = new ReadLogEntry(tailOffset + 2, readSnapshotId, emptyBatch);
    when(mockLogReader.readNext()).thenReturn(readLogEntry1).thenReturn(readLogEntry2).thenReturn(null);
    BatchSender mockBatchSender = mock(BatchSender.class);
    AtomicLong ingestSnapshotId = new AtomicLong(10L);
    IngestProcessor ingestProcessor = new IngestProcessor(configs, mockLogService, mockBatchSender, queueId, ingestSnapshotId, new MetricsCollector(configs));
    ingestProcessor.setTailOffset(tailOffset);
    ingestProcessor.start();
    verify(mockBatchSender, timeout(5000L)).asyncSendWithRetry(eq(""), eq(queueId), eq(readSnapshotId), eq(tailOffset + 1), any());
    verify(mockBatchSender, timeout(5000L)).asyncSendWithRetry(eq(""), eq(queueId), eq(readSnapshotId), eq(tailOffset + 2), any());
    verify(mockLogReader).close();
    String requestId = "test_ingest_processor";
    IngestCallback mockIngestCallback = mock(IngestCallback.class);
    ingestProcessor.ingestBatch(requestId, emptyBatch, mockIngestCallback);
    verify(mockBatchSender, timeout(5000L)).asyncSendWithRetry(requestId, queueId, ingestSnapshotId.get(), tailOffset + 3, emptyBatch);
    verify(mockIngestCallback, timeout(5000L)).onSuccess(ingestSnapshotId.get());
    ingestProcessor.stop();
    verify(mockLogWriter, timeout(5000L)).close();
}

Example 3 with ReadLogEntry

use of com.alibaba.graphscope.groot.wal.ReadLogEntry in project GraphScope by alibaba.

the class KafkaLogReader method readNext.

@Override
public ReadLogEntry readNext() {
    if (this.nextReadOffset == this.latestOffset) {
        return null;
    }
    while (this.iterator == null || !this.iterator.hasNext()) {
        ConsumerRecords<LogEntry, LogEntry> consumerRecords = this.consumer.poll(Duration.ofMillis(100L));
        if (consumerRecords == null || consumerRecords.isEmpty()) {
            logger.info("polled nothing from Kafka. nextReadOffset is [" + this.nextReadOffset + "]");
            continue;
        }
        this.iterator = consumerRecords.iterator();
    }
    ConsumerRecord<LogEntry, LogEntry> record = iterator.next();
    this.nextReadOffset = record.offset() + 1;
    LogEntry v = record.value();
    return new ReadLogEntry(record.offset(), v);
}

Example 4 with ReadLogEntry

use of com.alibaba.graphscope.groot.wal.ReadLogEntry in project GraphScope by alibaba.

the class IngestProcessor method replayWAL.

private void replayWAL(long tailOffset) throws IOException {
    long replayFrom = tailOffset + 1;
    logger.info("replay WAL of queue#[" + this.queueId + "] from offset [" + replayFrom + "]");
    LogReader logReader = this.logService.createReader(this.queueId, replayFrom);
    ReadLogEntry readLogEntry;
    int replayCount = 0;
    while (!shouldStop && (readLogEntry = logReader.readNext()) != null) {
        long offset = readLogEntry.getOffset();
        LogEntry logEntry = readLogEntry.getLogEntry();
        long snapshotId = logEntry.getSnapshotId();
        OperationBatch operationBatch = logEntry.getOperationBatch();
        this.batchSender.asyncSendWithRetry("", this.queueId, snapshotId, offset, operationBatch);
        replayCount++;
    }
    try {
        logReader.close();
    } catch (IOException e) {
        logger.warn("close logReader failed", e);
    }
    logger.info("replayWAL finished. total replayed [" + replayCount + "] records");
}