Examples with SnapshotInfo com.alibaba.graphscope.groot.coordinator.SnapshotInfo used on opensource projects

Example 1 with SnapshotInfo

use of com.alibaba.graphscope.groot.coordinator.SnapshotInfo in project GraphScope by alibaba.

the class SnapshotManagerTest method testSnapshotManager.

@Test
void testSnapshotManager() throws IOException, InterruptedException {
    Configs configs = Configs.newBuilder().put(CommonConfig.INGESTOR_QUEUE_COUNT.getKey(), "1").put(CommonConfig.STORE_NODE_COUNT.getKey(), "2").put(CommonConfig.FRONTEND_NODE_COUNT.getKey(), "1").put(CommonConfig.INGESTOR_NODE_COUNT.getKey(), "1").put(CoordinatorConfig.SNAPSHOT_INCREASE_INTERVAL_MS.getKey(), "1000").put(CoordinatorConfig.OFFSETS_PERSIST_INTERVAL_MS.getKey(), "1000").build();
    long querySnapshotId = 10L;
    long writeSnapshotId = 12L;
    List<Long> queueOffsets = Arrays.asList(50L);
    long commitSnapshotId1 = 11L;
    long commitSnapshotId2 = 12L;
    List<Long> commitQueueOffsets1 = Arrays.asList(60L);
    List<Long> commitQueueOffsets2 = Arrays.asList(70L);
    ObjectMapper objectMapper = new ObjectMapper();
    MetaStore mockMetaStore = mock(MetaStore.class);
    when(mockMetaStore.exists(anyString())).thenReturn(true);
    when(mockMetaStore.read(QUERY_SNAPSHOT_INFO_PATH)).thenReturn(objectMapper.writeValueAsBytes(new SnapshotInfo(querySnapshotId, querySnapshotId)));
    when(mockMetaStore.read(WRITE_SNAPSHOT_ID_PATH)).thenReturn(objectMapper.writeValueAsBytes(writeSnapshotId));
    when(mockMetaStore.read(QUEUE_OFFSETS_PATH)).thenReturn(objectMapper.writeValueAsBytes(queueOffsets));
    CountDownLatch updateWriteSnapshotLatch = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        updateWriteSnapshotLatch.countDown();
        return null;
    }).when(mockMetaStore).write(WRITE_SNAPSHOT_ID_PATH, objectMapper.writeValueAsBytes(writeSnapshotId + 1));
    CountDownLatch updateQueueOffsetLatch = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        updateQueueOffsetLatch.countDown();
        return null;
    }).when(mockMetaStore).write(QUEUE_OFFSETS_PATH, objectMapper.writeValueAsBytes(commitQueueOffsets1));
    IngestorWriteSnapshotIdNotifier mockWriteSnapshotIdNotifier = mock(IngestorWriteSnapshotIdNotifier.class);
    CountDownLatch updateIngestorLatch = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        updateIngestorLatch.countDown();
        return null;
    }).when(mockWriteSnapshotIdNotifier).notifyWriteSnapshotIdChanged(writeSnapshotId + 1);
    LogService mockLogService = mock(LogService.class);
    SnapshotManager snapshotManager = new SnapshotManager(configs, mockMetaStore, mockLogService, mockWriteSnapshotIdNotifier);
    snapshotManager.start();
    assertEquals(snapshotManager.getQueueOffsets(), queueOffsets);
    assertTrue(updateWriteSnapshotLatch.await(5L, TimeUnit.SECONDS));
    assertTrue(updateIngestorLatch.await(5L, TimeUnit.SECONDS));
    snapshotManager.commitSnapshotId(0, commitSnapshotId1, 10L, commitQueueOffsets1);
    snapshotManager.commitSnapshotId(1, commitSnapshotId2, 10L, commitQueueOffsets2);
    verify(mockMetaStore).write(QUERY_SNAPSHOT_INFO_PATH, objectMapper.writeValueAsBytes(new SnapshotInfo(commitSnapshotId1, 10L)));
    assertTrue(updateQueueOffsetLatch.await(5L, TimeUnit.SECONDS));
    assertEquals(snapshotManager.getQueueOffsets(), commitQueueOffsets1);
    snapshotManager.stop();
}

Example 2 with SnapshotInfo

use of com.alibaba.graphscope.groot.coordinator.SnapshotInfo in project GraphScope by alibaba.

the class WriterAgent method asyncCommit.

private void asyncCommit() {
    SnapshotInfo snapshotInfo = this.availSnapshotInfoRef.get();
    long availSnapshotId = snapshotInfo.getSnapshotId();
    if (availSnapshotId > this.lastCommitSnapshotId) {
        long ddlSnapshotId = snapshotInfo.getDdlSnapshotId();
        List<Long> queueOffsets = new ArrayList<>(this.consumedQueueOffsets);
        try {
            logger.debug("commit snapshotId [" + availSnapshotId + "], last DDL snapshotId [" + ddlSnapshotId + "]");
            this.snapshotCommitter.commitSnapshotId(this.storeId, availSnapshotId, ddlSnapshotId, queueOffsets);
            this.lastCommitSnapshotId = availSnapshotId;
        } catch (Exception e) {
            logger.warn("commit failed. snapshotId [" + availSnapshotId + "], queueOffsets [" + queueOffsets + "]. will ignore", e);
        }
    }
}

Example 3 with SnapshotInfo

use of com.alibaba.graphscope.groot.coordinator.SnapshotInfo in project GraphScope by alibaba.

the class WriterAgent method processBatches.

private void processBatches() {
    while (!shouldStop) {
        try {
            long beforePollNano = System.nanoTime();
            StoreDataBatch storeDataBatch = this.bufferQueue.poll();
            long afterPollNano = System.nanoTime();
            long pollNano = afterPollNano - beforePollNano;
            this.totalPollLatencyNano += pollNano;
            this.maxPollLatencyNano.updateAndGet(curMax -> (pollNano > curMax) ? pollNano : curMax);
            if (storeDataBatch == null) {
                continue;
            }
            long batchSnapshotId = storeDataBatch.getSnapshotId();
            logger.debug("polled one batch [" + batchSnapshotId + "]");
            boolean hasDdl = writeEngineWithRetry(storeDataBatch);
            int writeCount = storeDataBatch.getSize();
            this.totalWrite += writeCount;
            if (this.consumeSnapshotId < batchSnapshotId) {
                SnapshotInfo availSnapshotInfo = this.availSnapshotInfoRef.get();
                long availDdlSnapshotId = availSnapshotInfo.getDdlSnapshotId();
                if (availDdlSnapshotId < this.consumeDdlSnapshotId) {
                    availDdlSnapshotId = this.consumeDdlSnapshotId;
                }
                long prevSnapshotId = batchSnapshotId - 1;
                long availSnapshotId = availSnapshotInfo.getSnapshotId();
                if (availSnapshotId < prevSnapshotId) {
                    availSnapshotId = prevSnapshotId;
                }
                this.consumeSnapshotId = batchSnapshotId;
                this.availSnapshotInfoRef.set(new SnapshotInfo(availSnapshotId, availDdlSnapshotId));
                this.commitExecutor.execute(() -> asyncCommit());
            }
            if (hasDdl) {
                this.consumeDdlSnapshotId = batchSnapshotId;
            }
            int queueId = storeDataBatch.getQueueId();
            long offset = storeDataBatch.getOffset();
            this.consumedQueueOffsets.set(queueId, offset);
        } catch (Exception e) {
            logger.error("error in processBatches, ignore", e);
        }
    }
}