use of com.alibaba.graphscope.groot.wal.LogReader in project GraphScope by alibaba.
the class KafkaWalTest method testLogService.
@Test
void testLogService() throws IOException {
Configs configs = Configs.newBuilder().put(KafkaConfig.KAFKA_SERVERS.getKey(), sharedKafkaTestResource.getKafkaConnectString()).put(KafkaConfig.KAKFA_TOPIC.getKey(), "test_logservice").put(CommonConfig.INGESTOR_QUEUE_COUNT.getKey(), "1").build();
LogService logService = new KafkaLogService(configs);
logService.init();
int queueId = 0;
long snapshotId = 1L;
LogWriter writer = logService.createWriter(queueId);
LogEntry logEntry = new LogEntry(snapshotId, OperationBatch.newBuilder().addOperationBlob(OperationBlob.MARKER_OPERATION_BLOB).build());
assertEquals(writer.append(logEntry), 0);
LogReader reader = logService.createReader(queueId, 0);
ReadLogEntry readLogEntry = reader.readNext();
reader.close();
assertAll(() -> assertEquals(readLogEntry.getOffset(), 0), () -> assertEquals(readLogEntry.getLogEntry().getSnapshotId(), snapshotId));
OperationBatch operationBatch = readLogEntry.getLogEntry().getOperationBatch();
assertEquals(operationBatch.getOperationCount(), 1);
assertEquals(operationBatch.getOperationBlob(0), OperationBlob.MARKER_OPERATION_BLOB);
assertEquals(writer.append(logEntry), 1);
assertEquals(writer.append(logEntry), 2);
assertEquals(writer.append(logEntry), 3);
LogReader readerTail = logService.createReader(queueId, 4);
assertNull(readerTail.readNext());
readerTail.close();
assertThrows(IllegalArgumentException.class, () -> logService.createReader(queueId, 5));
logService.deleteBeforeOffset(queueId, 2);
assertThrows(IllegalArgumentException.class, () -> logService.createReader(queueId, 1));
writer.close();
logService.destroy();
}
use of com.alibaba.graphscope.groot.wal.LogReader in project GraphScope by alibaba.
the class IngestProcessorTest method testIngestProcessor.
@Test
void testIngestProcessor() throws IOException {
long tailOffset = 50L;
int queueId = 0;
Configs configs = Configs.newBuilder().build();
LogService mockLogService = mock(LogService.class);
LogReader mockLogReader = mock(LogReader.class);
when(mockLogService.createReader(queueId, tailOffset + 1)).thenReturn(mockLogReader);
LogWriter mockLogWriter = mock(LogWriter.class);
when(mockLogService.createWriter(queueId)).thenReturn(mockLogWriter);
when(mockLogWriter.append(any())).thenReturn(tailOffset + 3);
OperationBatch emptyBatch = OperationBatch.newBuilder().build();
long readSnapshotId = 5L;
ReadLogEntry readLogEntry1 = new ReadLogEntry(tailOffset + 1, readSnapshotId, emptyBatch);
ReadLogEntry readLogEntry2 = new ReadLogEntry(tailOffset + 2, readSnapshotId, emptyBatch);
when(mockLogReader.readNext()).thenReturn(readLogEntry1).thenReturn(readLogEntry2).thenReturn(null);
BatchSender mockBatchSender = mock(BatchSender.class);
AtomicLong ingestSnapshotId = new AtomicLong(10L);
IngestProcessor ingestProcessor = new IngestProcessor(configs, mockLogService, mockBatchSender, queueId, ingestSnapshotId, new MetricsCollector(configs));
ingestProcessor.setTailOffset(tailOffset);
ingestProcessor.start();
verify(mockBatchSender, timeout(5000L)).asyncSendWithRetry(eq(""), eq(queueId), eq(readSnapshotId), eq(tailOffset + 1), any());
verify(mockBatchSender, timeout(5000L)).asyncSendWithRetry(eq(""), eq(queueId), eq(readSnapshotId), eq(tailOffset + 2), any());
verify(mockLogReader).close();
String requestId = "test_ingest_processor";
IngestCallback mockIngestCallback = mock(IngestCallback.class);
ingestProcessor.ingestBatch(requestId, emptyBatch, mockIngestCallback);
verify(mockBatchSender, timeout(5000L)).asyncSendWithRetry(requestId, queueId, ingestSnapshotId.get(), tailOffset + 3, emptyBatch);
verify(mockIngestCallback, timeout(5000L)).onSuccess(ingestSnapshotId.get());
ingestProcessor.stop();
verify(mockLogWriter, timeout(5000L)).close();
}
use of com.alibaba.graphscope.groot.wal.LogReader in project GraphScope by alibaba.
the class SnapshotManager method recover.
private void recover() throws IOException {
checkMetaPath(QUERY_SNAPSHOT_INFO_PATH);
checkMetaPath(WRITE_SNAPSHOT_ID_PATH);
checkMetaPath(QUEUE_OFFSETS_PATH);
byte[] querySnapshotInfoBytes = this.metaStore.read(QUERY_SNAPSHOT_INFO_PATH);
SnapshotInfo recoveredQuerySnapshotInfo = this.objectMapper.readValue(querySnapshotInfoBytes, SnapshotInfo.class);
byte[] writeSnapshotIdBytes = this.metaStore.read(WRITE_SNAPSHOT_ID_PATH);
long recoveredWriteSnapshotId = this.objectMapper.readValue(writeSnapshotIdBytes, Long.class);
if (recoveredQuerySnapshotInfo.getSnapshotId() > recoveredWriteSnapshotId) {
throw new IllegalStateException("recovered querySnapshotInfo [" + recoveredQuerySnapshotInfo + "] > writeSnapshotId [" + recoveredWriteSnapshotId + "]");
}
byte[] queueOffsetsBytes = this.metaStore.read(QUEUE_OFFSETS_PATH);
List<Long> recoveredQueueOffsets = this.objectMapper.readValue(queueOffsetsBytes, new TypeReference<List<Long>>() {
});
logger.info("recovered queue offsets [" + recoveredQueueOffsets + "]");
if (recoveredQueueOffsets.size() != this.queueCount) {
throw new IllegalStateException("recovered queueCount [" + recoveredQueueOffsets.size() + "], but expect queueCount [" + this.queueCount + "]");
}
for (int i = 0; i < this.queueCount; i++) {
long recoveredOffset = recoveredQueueOffsets.get(i);
LogReader reader = null;
try {
reader = logService.createReader(i, recoveredOffset + 1);
} catch (Exception e) {
throw new IOException("recovered queue [" + i + "] offset [" + recoveredOffset + "] is not available", e);
} finally {
if (reader != null) {
reader.close();
}
}
}
this.querySnapshotInfo = recoveredQuerySnapshotInfo;
this.writeSnapshotId = recoveredWriteSnapshotId;
this.queueOffsetsRef = new AtomicReference<>(recoveredQueueOffsets);
}
use of com.alibaba.graphscope.groot.wal.LogReader in project GraphScope by alibaba.
the class IngestProcessor method replayWAL.
private void replayWAL(long tailOffset) throws IOException {
long replayFrom = tailOffset + 1;
logger.info("replay WAL of queue#[" + this.queueId + "] from offset [" + replayFrom + "]");
LogReader logReader = this.logService.createReader(this.queueId, replayFrom);
ReadLogEntry readLogEntry;
int replayCount = 0;
while (!shouldStop && (readLogEntry = logReader.readNext()) != null) {
long offset = readLogEntry.getOffset();
LogEntry logEntry = readLogEntry.getLogEntry();
long snapshotId = logEntry.getSnapshotId();
OperationBatch operationBatch = logEntry.getOperationBatch();
this.batchSender.asyncSendWithRetry("", this.queueId, snapshotId, offset, operationBatch);
replayCount++;
}
try {
logReader.close();
} catch (IOException e) {
logger.warn("close logReader failed", e);
}
logger.info("replayWAL finished. total replayed [" + replayCount + "] records");
}
Aggregations