Examples with LogService com.alibaba.graphscope.groot.wal.LogService used on opensource projects

Example 1 with LogService

use of com.alibaba.graphscope.groot.wal.LogService in project GraphScope by alibaba.

the class KafkaWalTest method testDoubleInit.

@Test
void testDoubleInit() {
    Configs configs = Configs.newBuilder().put(KafkaConfig.KAFKA_SERVERS.getKey(), sharedKafkaTestResource.getKafkaConnectString()).put(KafkaConfig.KAKFA_TOPIC.getKey(), "test_double_init").put(CommonConfig.INGESTOR_QUEUE_COUNT.getKey(), "1").build();
    LogService logService = new KafkaLogService(configs);
    logService.init();
    assertThrows(Exception.class, () -> logService.init());
    logService.destroy();
}

Example 2 with LogService

use of com.alibaba.graphscope.groot.wal.LogService in project GraphScope by alibaba.

the class SnapshotManagerTest method testSnapshotManager.

@Test
void testSnapshotManager() throws IOException, InterruptedException {
    Configs configs = Configs.newBuilder().put(CommonConfig.INGESTOR_QUEUE_COUNT.getKey(), "1").put(CommonConfig.STORE_NODE_COUNT.getKey(), "2").put(CommonConfig.FRONTEND_NODE_COUNT.getKey(), "1").put(CommonConfig.INGESTOR_NODE_COUNT.getKey(), "1").put(CoordinatorConfig.SNAPSHOT_INCREASE_INTERVAL_MS.getKey(), "1000").put(CoordinatorConfig.OFFSETS_PERSIST_INTERVAL_MS.getKey(), "1000").build();
    long querySnapshotId = 10L;
    long writeSnapshotId = 12L;
    List<Long> queueOffsets = Arrays.asList(50L);
    long commitSnapshotId1 = 11L;
    long commitSnapshotId2 = 12L;
    List<Long> commitQueueOffsets1 = Arrays.asList(60L);
    List<Long> commitQueueOffsets2 = Arrays.asList(70L);
    ObjectMapper objectMapper = new ObjectMapper();
    MetaStore mockMetaStore = mock(MetaStore.class);
    when(mockMetaStore.exists(anyString())).thenReturn(true);
    when(mockMetaStore.read(QUERY_SNAPSHOT_INFO_PATH)).thenReturn(objectMapper.writeValueAsBytes(new SnapshotInfo(querySnapshotId, querySnapshotId)));
    when(mockMetaStore.read(WRITE_SNAPSHOT_ID_PATH)).thenReturn(objectMapper.writeValueAsBytes(writeSnapshotId));
    when(mockMetaStore.read(QUEUE_OFFSETS_PATH)).thenReturn(objectMapper.writeValueAsBytes(queueOffsets));
    CountDownLatch updateWriteSnapshotLatch = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        updateWriteSnapshotLatch.countDown();
        return null;
    }).when(mockMetaStore).write(WRITE_SNAPSHOT_ID_PATH, objectMapper.writeValueAsBytes(writeSnapshotId + 1));
    CountDownLatch updateQueueOffsetLatch = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        updateQueueOffsetLatch.countDown();
        return null;
    }).when(mockMetaStore).write(QUEUE_OFFSETS_PATH, objectMapper.writeValueAsBytes(commitQueueOffsets1));
    IngestorWriteSnapshotIdNotifier mockWriteSnapshotIdNotifier = mock(IngestorWriteSnapshotIdNotifier.class);
    CountDownLatch updateIngestorLatch = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        updateIngestorLatch.countDown();
        return null;
    }).when(mockWriteSnapshotIdNotifier).notifyWriteSnapshotIdChanged(writeSnapshotId + 1);
    LogService mockLogService = mock(LogService.class);
    SnapshotManager snapshotManager = new SnapshotManager(configs, mockMetaStore, mockLogService, mockWriteSnapshotIdNotifier);
    snapshotManager.start();
    assertEquals(snapshotManager.getQueueOffsets(), queueOffsets);
    assertTrue(updateWriteSnapshotLatch.await(5L, TimeUnit.SECONDS));
    assertTrue(updateIngestorLatch.await(5L, TimeUnit.SECONDS));
    snapshotManager.commitSnapshotId(0, commitSnapshotId1, 10L, commitQueueOffsets1);
    snapshotManager.commitSnapshotId(1, commitSnapshotId2, 10L, commitQueueOffsets2);
    verify(mockMetaStore).write(QUERY_SNAPSHOT_INFO_PATH, objectMapper.writeValueAsBytes(new SnapshotInfo(commitSnapshotId1, 10L)));
    assertTrue(updateQueueOffsetLatch.await(5L, TimeUnit.SECONDS));
    assertEquals(snapshotManager.getQueueOffsets(), commitQueueOffsets1);
    snapshotManager.stop();
}

Example 3 with LogService

use of com.alibaba.graphscope.groot.wal.LogService in project GraphScope by alibaba.

the class KafkaWalTest method testLogService.

@Test
void testLogService() throws IOException {
    Configs configs = Configs.newBuilder().put(KafkaConfig.KAFKA_SERVERS.getKey(), sharedKafkaTestResource.getKafkaConnectString()).put(KafkaConfig.KAKFA_TOPIC.getKey(), "test_logservice").put(CommonConfig.INGESTOR_QUEUE_COUNT.getKey(), "1").build();
    LogService logService = new KafkaLogService(configs);
    logService.init();
    int queueId = 0;
    long snapshotId = 1L;
    LogWriter writer = logService.createWriter(queueId);
    LogEntry logEntry = new LogEntry(snapshotId, OperationBatch.newBuilder().addOperationBlob(OperationBlob.MARKER_OPERATION_BLOB).build());
    assertEquals(writer.append(logEntry), 0);
    LogReader reader = logService.createReader(queueId, 0);
    ReadLogEntry readLogEntry = reader.readNext();
    reader.close();
    assertAll(() -> assertEquals(readLogEntry.getOffset(), 0), () -> assertEquals(readLogEntry.getLogEntry().getSnapshotId(), snapshotId));
    OperationBatch operationBatch = readLogEntry.getLogEntry().getOperationBatch();
    assertEquals(operationBatch.getOperationCount(), 1);
    assertEquals(operationBatch.getOperationBlob(0), OperationBlob.MARKER_OPERATION_BLOB);
    assertEquals(writer.append(logEntry), 1);
    assertEquals(writer.append(logEntry), 2);
    assertEquals(writer.append(logEntry), 3);
    LogReader readerTail = logService.createReader(queueId, 4);
    assertNull(readerTail.readNext());
    readerTail.close();
    assertThrows(IllegalArgumentException.class, () -> logService.createReader(queueId, 5));
    logService.deleteBeforeOffset(queueId, 2);
    assertThrows(IllegalArgumentException.class, () -> logService.createReader(queueId, 1));
    writer.close();
    logService.destroy();
}

Example 4 with LogService

use of com.alibaba.graphscope.groot.wal.LogService in project GraphScope by alibaba.

the class KafkaWalTest method testDoubleDestroy.

@Test
void testDoubleDestroy() {
    Configs configs = Configs.newBuilder().put(KafkaConfig.KAFKA_SERVERS.getKey(), sharedKafkaTestResource.getKafkaConnectString()).put(KafkaConfig.KAKFA_TOPIC.getKey(), "test_double_destroy").put(CommonConfig.INGESTOR_QUEUE_COUNT.getKey(), "1").build();
    LogService logService = new KafkaLogService(configs);
    logService.init();
    logService.destroy();
    assertThrows(Exception.class, () -> logService.destroy());
}

Example 5 with LogService

use of com.alibaba.graphscope.groot.wal.LogService in project GraphScope by alibaba.

the class LogRecyclerTest method testRecycler.

@Test
void testRecycler() throws IOException {
    Configs configs = Configs.newBuilder().put(CoordinatorConfig.LOG_RECYCLE_INTERVAL_SECOND.getKey(), "1").build();
    SnapshotManager mockSnapshotManager = mock(SnapshotManager.class);
    when(mockSnapshotManager.getQueueOffsets()).thenReturn(Arrays.asList(1L, 2L, 3L));
    LogService mockLogService = mock(LogService.class);
    LogRecycler logRecycler = new LogRecycler(configs, mockLogService, mockSnapshotManager);
    CountDownLatch latch1 = new CountDownLatch(1);
    CountDownLatch latch2 = new CountDownLatch(1);
    CountDownLatch latch3 = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        latch1.countDown();
        return null;
    }).when(mockLogService).deleteBeforeOffset(0, 1L);
    doAnswer(invocationOnMock -> {
        latch2.countDown();
        return null;
    }).when(mockLogService).deleteBeforeOffset(1, 2L);
    doAnswer(invocationOnMock -> {
        latch3.countDown();
        return null;
    }).when(mockLogService).deleteBeforeOffset(2, 3L);
    logRecycler.start();
    assertAll(() -> assertTrue(latch1.await(5L, TimeUnit.SECONDS)), () -> assertTrue(latch2.await(5L, TimeUnit.SECONDS)), () -> assertTrue(latch3.await(5L, TimeUnit.SECONDS)));
    logRecycler.stop();
}