Examples with StoreBackupId com.alibaba.graphscope.groot.store.StoreBackupId used on opensource projects

Example 1 with StoreBackupId

use of com.alibaba.graphscope.groot.store.StoreBackupId in project GraphScope by alibaba.

the class BackupManagerTest method testBackupManager.

@Test
void testBackupManager() throws IOException, InterruptedException {
    // init config
    Configs configs = Configs.newBuilder().put(CommonConfig.STORE_NODE_COUNT.getKey(), "2").put(BackupConfig.BACKUP_ENABLE.getKey(), "true").put(BackupConfig.BACKUP_CREATION_BUFFER_MAX_COUNT.getKey(), "4").put(BackupConfig.BACKUP_GC_INTERVAL_HOURS.getKey(), "24").put(BackupConfig.BACKUP_AUTO_SUBMIT.getKey(), "false").put(BackupConfig.BACKUP_AUTO_SUBMIT_INTERVAL_HOURS.getKey(), "24").build();
    // init data
    long querySnapshotId = 10L;
    GraphDef graphDef = GraphDef.newBuilder().setVersion(10L).build();
    SnapshotWithSchema snapshotWithSchema = new SnapshotWithSchema(querySnapshotId, graphDef);
    List<Long> queueOffsets = new ArrayList<>();
    Map<Integer, Integer> partitionToBackupId1 = new HashMap<>();
    partitionToBackupId1.put(0, 1);
    partitionToBackupId1.put(1, 1);
    Map<Integer, Integer> partitionToBackupId2 = new HashMap<>();
    partitionToBackupId2.put(0, 2);
    partitionToBackupId2.put(1, 2);
    BackupInfo backupInfo1 = new BackupInfo(1, querySnapshotId, graphDef.toProto().toByteArray(), queueOffsets, partitionToBackupId1);
    BackupInfo backupInfo2 = new BackupInfo(2, querySnapshotId, graphDef.toProto().toByteArray(), queueOffsets, partitionToBackupId2);
    // mock MetaStore behaviours
    ObjectMapper objectMapper = new ObjectMapper();
    MetaStore mockMetaStore = mock(MetaStore.class);
    when(mockMetaStore.exists(anyString())).thenReturn(true);
    when(mockMetaStore.read(GLOBAL_BACKUP_ID_PATH)).thenReturn(objectMapper.writeValueAsBytes(0));
    when(mockMetaStore.read(BACKUP_INFO_PATH)).thenReturn(objectMapper.writeValueAsBytes(new ArrayList<BackupInfo>()));
    // mock MetaService behaviours
    MetaService mockMetaService = mock(MetaService.class);
    when(mockMetaService.getPartitionCount()).thenReturn(2);
    when(mockMetaService.getStoreIdByPartition(0)).thenReturn(0);
    when(mockMetaService.getStoreIdByPartition(1)).thenReturn(1);
    // mock SnapshotManager behaviours
    SnapshotManager mockSnapshotManager = mock(SnapshotManager.class);
    when(mockSnapshotManager.getQueueOffsets()).thenReturn(queueOffsets);
    // mock SchemaManager
    SchemaManager mockSchemaManager = mock(SchemaManager.class);
    when(mockSchemaManager.getGraphDef()).thenReturn(graphDef);
    // mock SnapshotCache
    SnapshotCache mockSnapshotCache = mock(SnapshotCache.class);
    when(mockSnapshotCache.getSnapshotWithSchema()).thenReturn(snapshotWithSchema);
    // mock StoreBackupTaskSender behaviours
    StoreBackupTaskSender mockStoreBackupTaskSender = mock(StoreBackupTaskSender.class);
    doAnswer(invocation -> {
        int partitionOrStoreId = invocation.getArgument(0);
        int globalBackupId = invocation.getArgument(1);
        CompletionCallback<StoreBackupId> callback = invocation.getArgument(2);
        StoreBackupId storeBackupId = new StoreBackupId(globalBackupId);
        storeBackupId.addPartitionBackupId(partitionOrStoreId, globalBackupId);
        callback.onCompleted(storeBackupId);
        return null;
    }).when(mockStoreBackupTaskSender).createStoreBackup(anyInt(), anyInt(), any());
    doAnswer(invocation -> {
        CompletionCallback<Void> callback = invocation.getArgument(3);
        callback.onCompleted(null);
        return null;
    }).when(mockStoreBackupTaskSender).restoreFromStoreBackup(anyInt(), any(), anyString(), any());
    doAnswer(invocation -> {
        CompletionCallback<Void> callback = invocation.getArgument(2);
        callback.onCompleted(null);
        return null;
    }).when(mockStoreBackupTaskSender).verifyStoreBackup(anyInt(), any(), any());
    BackupManager backupManager = new BackupManager(configs, mockMetaService, mockMetaStore, mockSnapshotManager, mockSchemaManager, mockSnapshotCache, mockStoreBackupTaskSender);
    backupManager.start();
    verify(mockSnapshotManager).addListener(any());
    // create the first backup
    CountDownLatch updateBackupIdLatch1 = new CountDownLatch(1);
    CountDownLatch updateBackupInfoByCreation1Latch = new CountDownLatch(1);
    doAnswer(invocation -> {
        updateBackupIdLatch1.countDown();
        return null;
    }).when(mockMetaStore).write(GLOBAL_BACKUP_ID_PATH, objectMapper.writeValueAsBytes(1));
    doAnswer(invocation -> {
        byte[] backupInfoBytes = invocation.getArgument(1);
        List<BackupInfo> backupInfoList = objectMapper.readValue(backupInfoBytes, new TypeReference<List<BackupInfo>>() {
        });
        assertEquals(backupInfoList.size(), 1);
        assertEquals(backupInfoList.get(0), backupInfo1);
        updateBackupInfoByCreation1Latch.countDown();
        return null;
    }).when(mockMetaStore).write(BACKUP_INFO_PATH, objectMapper.writeValueAsBytes(Collections.singletonList(backupInfo1)));
    int backupId1 = backupManager.createNewBackup();
    assertEquals(backupId1, 1);
    assertTrue(updateBackupIdLatch1.await(5L, TimeUnit.SECONDS));
    assertTrue(updateBackupInfoByCreation1Latch.await(5L, TimeUnit.SECONDS));
    // create the second backup
    CountDownLatch updateBackupIdLatch2 = new CountDownLatch(1);
    CountDownLatch updateBackupInfoByCreation2Latch = new CountDownLatch(1);
    doAnswer(invocation -> {
        updateBackupIdLatch2.countDown();
        return null;
    }).when(mockMetaStore).write(GLOBAL_BACKUP_ID_PATH, objectMapper.writeValueAsBytes(2));
    doAnswer(invocation -> {
        byte[] backupInfoBytes = invocation.getArgument(1);
        List<BackupInfo> backupInfoList = objectMapper.readValue(backupInfoBytes, new TypeReference<List<BackupInfo>>() {
        });
        backupInfoList.sort(new Comparator<BackupInfo>() {

            @Override
            public int compare(BackupInfo o1, BackupInfo o2) {
                return o1.getGlobalBackupId() - o2.getGlobalBackupId();
            }
        });
        assertEquals(backupInfoList.size(), 2);
        assertEquals(backupInfoList.get(0), backupInfo1);
        assertEquals(backupInfoList.get(1), backupInfo2);
        updateBackupInfoByCreation2Latch.countDown();
        return null;
    }).when(mockMetaStore).write(BACKUP_INFO_PATH, objectMapper.writeValueAsBytes(Arrays.asList(backupInfo1, backupInfo2)));
    int backupId2 = backupManager.createNewBackup();
    assertEquals(backupId2, 2);
    assertTrue(updateBackupIdLatch2.await(5L, TimeUnit.SECONDS));
    assertTrue(updateBackupInfoByCreation2Latch.await(5L, TimeUnit.SECONDS));
    // get backup info list and check
    assertEquals(backupManager.getBackupInfoList().size(), 2);
    // verify backups
    try {
        backupManager.verifyBackup(backupId1);
        backupManager.verifyBackup(backupId2);
    } catch (Exception e) {
        fail("should not have thrown any exception during backup verification");
    }
    // restore from the second backup
    try {
        backupManager.restoreFromBackup(backupId2, "restore_meta", "restore_store");
        assertTrue(Files.exists(Paths.get("restore_meta", "query_snapshot_id")));
        assertTrue(Files.exists(Paths.get("restore_meta", "graph_def_proto_bytes")));
        assertTrue(Files.exists(Paths.get("restore_meta", "queue_offsets")));
    } catch (Exception e) {
        fail("should not have thrown any exception during backup restoring");
    } finally {
        FileUtils.deleteDirectory(new File("restore_meta"));
    }
    // purge 1 old backup
    CountDownLatch updateBackupInfoByPurgingLatch = new CountDownLatch(1);
    doAnswer(invocation -> {
        byte[] backupInfoBytes = invocation.getArgument(1);
        List<BackupInfo> backupInfoList = objectMapper.readValue(backupInfoBytes, new TypeReference<List<BackupInfo>>() {
        });
        assertEquals(backupInfoList.size(), 1);
        assertEquals(backupInfoList.get(0), backupInfo2);
        updateBackupInfoByPurgingLatch.countDown();
        return null;
    }).when(mockMetaStore).write(BACKUP_INFO_PATH, objectMapper.writeValueAsBytes(Collections.singletonList(backupInfo2)));
    backupManager.purgeOldBackups(1);
    assertTrue(updateBackupInfoByPurgingLatch.await(5L, TimeUnit.SECONDS));
    // get backup info list and check
    assertEquals(backupManager.getBackupInfoList().size(), 1);
    // delete the remaining backup '2'
    CountDownLatch updateBackupInfoByDeletionLatch = new CountDownLatch(1);
    doAnswer(invocation -> {
        byte[] backupInfoBytes = invocation.getArgument(1);
        List<BackupInfo> backupInfoList = objectMapper.readValue(backupInfoBytes, new TypeReference<List<BackupInfo>>() {
        });
        assertTrue(backupInfoList.isEmpty());
        updateBackupInfoByDeletionLatch.countDown();
        return null;
    }).when(mockMetaStore).write(BACKUP_INFO_PATH, objectMapper.writeValueAsBytes(new ArrayList<BackupInfo>()));
    backupManager.deleteBackup(2);
    assertTrue(updateBackupInfoByDeletionLatch.await(5L, TimeUnit.SECONDS));
    // get backup info list and check
    assertTrue(backupManager.getBackupInfoList().isEmpty());
    backupManager.stop();
    verify(mockSnapshotManager).removeListener(any());
}

Example 2 with StoreBackupId

use of com.alibaba.graphscope.groot.store.StoreBackupId in project GraphScope by alibaba.

the class CoordinatorRpcTest method testStoreBackupClient.

@Test
void testStoreBackupClient() {
    StoreBackupGrpc.StoreBackupStub stub = mock(StoreBackupGrpc.StoreBackupStub.class);
    StoreBackupClient client = new StoreBackupClient(stub);
    StoreBackupId storeBackupId = new StoreBackupId(3);
    storeBackupId.addPartitionBackupId(0, 3);
    storeBackupId.addPartitionBackupId(1, 3);
    Map<Integer, List<Integer>> readyPartitionBackupIds = new HashMap<>();
    readyPartitionBackupIds.put(6, new ArrayList<>());
    String storeRestoreRootPath = "store_restore_path";
    doAnswer(invocation -> {
        CreateStoreBackupRequest request = invocation.getArgument(0);
        assertEquals(request.getGlobalBackupId(), 3);
        StreamObserver<CreateStoreBackupResponse> observer = invocation.getArgument(1);
        observer.onNext(CreateStoreBackupResponse.newBuilder().setStoreBackupId(storeBackupId.toProto()).build());
        observer.onError(null);
        return null;
    }).when(stub).createStoreBackup(any(), any());
    doAnswer(invocation -> {
        ClearUnavailableStoreBackupsRequest request = invocation.getArgument(0);
        assertEquals(request.getPartitionToReadyBackupIdsCount(), 1);
        assertTrue(request.getPartitionToReadyBackupIdsMap().containsKey(6));
        StreamObserver<ClearUnavailableStoreBackupsResponse> observer = invocation.getArgument(1);
        observer.onNext(ClearUnavailableStoreBackupsResponse.newBuilder().build());
        observer.onError(null);
        return null;
    }).when(stub).clearUnavailableStoreBackups(any(), any());
    doAnswer(invocation -> {
        RestoreFromStoreBackupRequest request = invocation.getArgument(0);
        assertEquals(StoreBackupId.parseProto(request.getStoreBackupId()), storeBackupId);
        assertEquals(request.getRestoreRootPath(), storeRestoreRootPath);
        StreamObserver<RestoreFromStoreBackupResponse> observer = invocation.getArgument(1);
        observer.onNext(RestoreFromStoreBackupResponse.newBuilder().build());
        observer.onError(null);
        return null;
    }).when(stub).restoreFromStoreBackup(any(), any());
    doAnswer(invocation -> {
        VerifyStoreBackupRequest request = invocation.getArgument(0);
        assertEquals(StoreBackupId.parseProto(request.getStoreBackupId()), storeBackupId);
        StreamObserver<VerifyStoreBackupResponse> observer = invocation.getArgument(1);
        observer.onNext(VerifyStoreBackupResponse.newBuilder().build());
        observer.onError(null);
        return null;
    }).when(stub).verifyStoreBackup(any(), any());
    CompletionCallback creationCallback = mock(CompletionCallback.class);
    client.createStoreBackup(3, creationCallback);
    verify(creationCallback).onCompleted(storeBackupId);
    verify(creationCallback).onError(null);
    CompletionCallback voidCallback = mock(CompletionCallback.class);
    client.clearUnavailableBackups(readyPartitionBackupIds, voidCallback);
    client.restoreFromStoreBackup(storeBackupId, storeRestoreRootPath, voidCallback);
    client.verifyStoreBackup(storeBackupId, voidCallback);
    verify(voidCallback, times(3)).onCompleted(null);
    verify(voidCallback, times(3)).onError(null);
}

Example 3 with StoreBackupId

use of com.alibaba.graphscope.groot.store.StoreBackupId in project GraphScope by alibaba.

the class BackupAgentTest method testBackupAgent.

@Test
void testBackupAgent() throws IOException {
    Configs configs = Configs.newBuilder().put(CommonConfig.NODE_IDX.getKey(), "0").put(BackupConfig.BACKUP_ENABLE.getKey(), "true").put(BackupConfig.STORE_BACKUP_THREAD_COUNT.getKey(), "2").build();
    StoreService mockStoreService = mock(StoreService.class);
    JnaGraphStore mockJnaStore0 = mock(JnaGraphStore.class);
    JnaGraphStore mockJnaStore1 = mock(JnaGraphStore.class);
    JnaGraphBackupEngine mockJnaBackupEngine0 = mock(JnaGraphBackupEngine.class);
    JnaGraphBackupEngine mockJnaBackupEngine1 = mock(JnaGraphBackupEngine.class);
    Map<Integer, GraphPartition> idToPartition = new HashMap<>();
    idToPartition.put(0, mockJnaStore0);
    idToPartition.put(1, mockJnaStore1);
    when(mockStoreService.getIdToPartition()).thenReturn(idToPartition);
    when(mockJnaStore0.openBackupEngine()).thenReturn(mockJnaBackupEngine0);
    when(mockJnaStore1.openBackupEngine()).thenReturn(mockJnaBackupEngine1);
    BackupAgent backupAgent = new BackupAgent(configs, mockStoreService);
    backupAgent.start();
    StoreBackupId storeBackupId = new StoreBackupId(5);
    storeBackupId.addPartitionBackupId(0, 7);
    storeBackupId.addPartitionBackupId(1, 6);
    Map<Integer, List<Integer>> readyPartitionBackupIds = new HashMap<>();
    readyPartitionBackupIds.put(0, Arrays.asList(2, 4, 6));
    readyPartitionBackupIds.put(1, Arrays.asList(3, 5, 7));
    when(mockJnaBackupEngine0.createNewPartitionBackup()).thenReturn(7);
    when(mockJnaBackupEngine1.createNewPartitionBackup()).thenReturn(6);
    CompletionCallback<StoreBackupId> createCallback = mock(CompletionCallback.class);
    backupAgent.createNewStoreBackup(5, createCallback);
    verify(createCallback, timeout(5000L)).onCompleted(storeBackupId);
    CompletionCallback<Void> verifyCallback = mock(CompletionCallback.class);
    backupAgent.verifyStoreBackup(storeBackupId, verifyCallback);
    verify(mockJnaBackupEngine0, timeout(5000L)).verifyPartitionBackup(7);
    verify(mockJnaBackupEngine1, timeout(5000L)).verifyPartitionBackup(6);
    verify(verifyCallback, timeout(5000L)).onCompleted(null);
    CompletionCallback<Void> clearCallback = mock(CompletionCallback.class);
    backupAgent.clearUnavailableStoreBackups(readyPartitionBackupIds, clearCallback);
    verify(mockJnaBackupEngine0, timeout(5000L)).partitionBackupGc(Arrays.asList(2, 4, 6));
    verify(mockJnaBackupEngine1, timeout(5000L)).partitionBackupGc(Arrays.asList(3, 5, 7));
    verify(clearCallback, timeout(5000L)).onCompleted(null);
    CompletionCallback<Void> restoreCallback = mock(CompletionCallback.class);
    backupAgent.restoreFromStoreBackup(storeBackupId, "restore_root", restoreCallback);
    verify(mockJnaBackupEngine0, timeout(5000L)).restoreFromPartitionBackup(7, Paths.get("restore_root", "0").toString());
    verify(mockJnaBackupEngine1, timeout(5000L)).restoreFromPartitionBackup(6, Paths.get("restore_root", "1").toString());
    verify(restoreCallback, timeout(5000L)).onCompleted(null);
    backupAgent.stop();
}

Example 4 with StoreBackupId

use of com.alibaba.graphscope.groot.store.StoreBackupId in project GraphScope by alibaba.

the class BackupManager method doBackupCreation.

private void doBackupCreation(int newGlobalBackupId) {
    List<Long> walOffsets = snapshotManager.getQueueOffsets();
    SnapshotWithSchema snapshotWithSchema = localSnapshotCache.getSnapshotWithSchema();
    Map<Integer, Integer> partitionToBackupId = new ConcurrentHashMap<>(graphPartitionCount);
    AtomicInteger counter = new AtomicInteger(storeNodeCount);
    AtomicBoolean finished = new AtomicBoolean(false);
    CompletableFuture<Void> future = new CompletableFuture<>();
    for (int sId = 0; sId < storeNodeCount; sId++) {
        storeBackupTaskSender.createStoreBackup(sId, newGlobalBackupId, new CompletionCallback<StoreBackupId>() {

            @Override
            public void onCompleted(StoreBackupId storeBackupId) {
                if (finished.get()) {
                    return;
                }
                partitionToBackupId.putAll(storeBackupId.getPartitionToBackupId());
                if (counter.decrementAndGet() == 0) {
                    if (partitionToBackupId.size() == graphPartitionCount) {
                        try {
                            addNewBackupInfo(new BackupInfo(newGlobalBackupId, snapshotWithSchema.getSnapshotId(), snapshotWithSchema.getGraphDef().toProto().toByteArray(), walOffsets, partitionToBackupId));
                            future.complete(null);
                        } catch (Exception e) {
                            future.completeExceptionally(new BackupException("failed to persist backup info of new" + " created backup #[" + newGlobalBackupId + "], " + e.getMessage()));
                        }
                    } else {
                        future.completeExceptionally(new BackupException("got incorrect number of partition backupIds" + " when creating backup #[" + newGlobalBackupId + "], got " + partitionToBackupId.size() + ", expect " + graphPartitionCount));
                    }
                }
            }

            @Override
            public void onError(Throwable t) {
                if (finished.getAndSet(true)) {
                    return;
                }
                future.completeExceptionally(t);
            }
        });
    }
    try {
        future.get();
        logger.info("new backup [" + newGlobalBackupId + "] created");
    } catch (Exception e) {
        logger.error("create new backup [" + newGlobalBackupId + "] failed", e);
    }
}

Example 5 with StoreBackupId

use of com.alibaba.graphscope.groot.store.StoreBackupId in project GraphScope by alibaba.

the class BackupManager method getStoreBackupIds.

private List<StoreBackupId> getStoreBackupIds(int globalBackupId) {
    List<StoreBackupId> storeBackupIds = new ArrayList<>(storeNodeCount);
    for (int sId = 0; sId < storeNodeCount; sId++) {
        storeBackupIds.add(new StoreBackupId(globalBackupId));
    }
    Map<Integer, Integer> partitionToBackupId = this.globalBackupIdToInfo.get(globalBackupId).getPartitionToBackupId();
    for (Map.Entry<Integer, Integer> entry : partitionToBackupId.entrySet()) {
        int partitionId = entry.getKey();
        int partitionBackupId = entry.getValue();
        int storeId = metaService.getStoreIdByPartition(partitionId);
        storeBackupIds.get(storeId).addPartitionBackupId(partitionId, partitionBackupId);
    }
    return storeBackupIds;
}