use of com.alibaba.graphscope.groot.wal.LogEntry in project GraphScope by alibaba.
the class KafkaWalTest method testLogService.
@Test
void testLogService() throws IOException {
Configs configs = Configs.newBuilder().put(KafkaConfig.KAFKA_SERVERS.getKey(), sharedKafkaTestResource.getKafkaConnectString()).put(KafkaConfig.KAKFA_TOPIC.getKey(), "test_logservice").put(CommonConfig.INGESTOR_QUEUE_COUNT.getKey(), "1").build();
LogService logService = new KafkaLogService(configs);
logService.init();
int queueId = 0;
long snapshotId = 1L;
LogWriter writer = logService.createWriter(queueId);
LogEntry logEntry = new LogEntry(snapshotId, OperationBatch.newBuilder().addOperationBlob(OperationBlob.MARKER_OPERATION_BLOB).build());
assertEquals(writer.append(logEntry), 0);
LogReader reader = logService.createReader(queueId, 0);
ReadLogEntry readLogEntry = reader.readNext();
reader.close();
assertAll(() -> assertEquals(readLogEntry.getOffset(), 0), () -> assertEquals(readLogEntry.getLogEntry().getSnapshotId(), snapshotId));
OperationBatch operationBatch = readLogEntry.getLogEntry().getOperationBatch();
assertEquals(operationBatch.getOperationCount(), 1);
assertEquals(operationBatch.getOperationBlob(0), OperationBlob.MARKER_OPERATION_BLOB);
assertEquals(writer.append(logEntry), 1);
assertEquals(writer.append(logEntry), 2);
assertEquals(writer.append(logEntry), 3);
LogReader readerTail = logService.createReader(queueId, 4);
assertNull(readerTail.readNext());
readerTail.close();
assertThrows(IllegalArgumentException.class, () -> logService.createReader(queueId, 5));
logService.deleteBeforeOffset(queueId, 2);
assertThrows(IllegalArgumentException.class, () -> logService.createReader(queueId, 1));
writer.close();
logService.destroy();
}
use of com.alibaba.graphscope.groot.wal.LogEntry in project GraphScope by alibaba.
the class KafkaLogReader method readNext.
@Override
public ReadLogEntry readNext() {
if (this.nextReadOffset == this.latestOffset) {
return null;
}
while (this.iterator == null || !this.iterator.hasNext()) {
ConsumerRecords<LogEntry, LogEntry> consumerRecords = this.consumer.poll(Duration.ofMillis(100L));
if (consumerRecords == null || consumerRecords.isEmpty()) {
logger.info("polled nothing from Kafka. nextReadOffset is [" + this.nextReadOffset + "]");
continue;
}
this.iterator = consumerRecords.iterator();
}
ConsumerRecord<LogEntry, LogEntry> record = iterator.next();
this.nextReadOffset = record.offset() + 1;
LogEntry v = record.value();
return new ReadLogEntry(record.offset(), v);
}
use of com.alibaba.graphscope.groot.wal.LogEntry in project GraphScope by alibaba.
the class IngestProcessor method replayWAL.
private void replayWAL(long tailOffset) throws IOException {
long replayFrom = tailOffset + 1;
logger.info("replay WAL of queue#[" + this.queueId + "] from offset [" + replayFrom + "]");
LogReader logReader = this.logService.createReader(this.queueId, replayFrom);
ReadLogEntry readLogEntry;
int replayCount = 0;
while (!shouldStop && (readLogEntry = logReader.readNext()) != null) {
long offset = readLogEntry.getOffset();
LogEntry logEntry = readLogEntry.getLogEntry();
long snapshotId = logEntry.getSnapshotId();
OperationBatch operationBatch = logEntry.getOperationBatch();
this.batchSender.asyncSendWithRetry("", this.queueId, snapshotId, offset, operationBatch);
replayCount++;
}
try {
logReader.close();
} catch (IOException e) {
logger.warn("close logReader failed", e);
}
logger.info("replayWAL finished. total replayed [" + replayCount + "] records");
}
Aggregations