Examples with Configs com.alibaba.maxgraph.common.config.Configs used on opensource projects

Example 21 with Configs

use of com.alibaba.maxgraph.common.config.Configs in project GraphScope by alibaba.

the class KafkaWalTest method testDoubleDestroy.

@Test
void testDoubleDestroy() {
    Configs configs = Configs.newBuilder().put(KafkaConfig.KAFKA_SERVERS.getKey(), sharedKafkaTestResource.getKafkaConnectString()).put(KafkaConfig.KAKFA_TOPIC.getKey(), "test_double_destroy").put(CommonConfig.INGESTOR_QUEUE_COUNT.getKey(), "1").build();
    LogService logService = new KafkaLogService(configs);
    logService.init();
    logService.destroy();
    assertThrows(Exception.class, () -> logService.destroy());
}

Example 22 with Configs

use of com.alibaba.maxgraph.common.config.Configs in project GraphScope by alibaba.

the class BackupManagerTest method testBackupManager.

@Test
void testBackupManager() throws IOException, InterruptedException {
    // init config
    Configs configs = Configs.newBuilder().put(CommonConfig.STORE_NODE_COUNT.getKey(), "2").put(BackupConfig.BACKUP_ENABLE.getKey(), "true").put(BackupConfig.BACKUP_CREATION_BUFFER_MAX_COUNT.getKey(), "4").put(BackupConfig.BACKUP_GC_INTERVAL_HOURS.getKey(), "24").put(BackupConfig.BACKUP_AUTO_SUBMIT.getKey(), "false").put(BackupConfig.BACKUP_AUTO_SUBMIT_INTERVAL_HOURS.getKey(), "24").build();
    // init data
    long querySnapshotId = 10L;
    GraphDef graphDef = GraphDef.newBuilder().setVersion(10L).build();
    SnapshotWithSchema snapshotWithSchema = new SnapshotWithSchema(querySnapshotId, graphDef);
    List<Long> queueOffsets = new ArrayList<>();
    Map<Integer, Integer> partitionToBackupId1 = new HashMap<>();
    partitionToBackupId1.put(0, 1);
    partitionToBackupId1.put(1, 1);
    Map<Integer, Integer> partitionToBackupId2 = new HashMap<>();
    partitionToBackupId2.put(0, 2);
    partitionToBackupId2.put(1, 2);
    BackupInfo backupInfo1 = new BackupInfo(1, querySnapshotId, graphDef.toProto().toByteArray(), queueOffsets, partitionToBackupId1);
    BackupInfo backupInfo2 = new BackupInfo(2, querySnapshotId, graphDef.toProto().toByteArray(), queueOffsets, partitionToBackupId2);
    // mock MetaStore behaviours
    ObjectMapper objectMapper = new ObjectMapper();
    MetaStore mockMetaStore = mock(MetaStore.class);
    when(mockMetaStore.exists(anyString())).thenReturn(true);
    when(mockMetaStore.read(GLOBAL_BACKUP_ID_PATH)).thenReturn(objectMapper.writeValueAsBytes(0));
    when(mockMetaStore.read(BACKUP_INFO_PATH)).thenReturn(objectMapper.writeValueAsBytes(new ArrayList<BackupInfo>()));
    // mock MetaService behaviours
    MetaService mockMetaService = mock(MetaService.class);
    when(mockMetaService.getPartitionCount()).thenReturn(2);
    when(mockMetaService.getStoreIdByPartition(0)).thenReturn(0);
    when(mockMetaService.getStoreIdByPartition(1)).thenReturn(1);
    // mock SnapshotManager behaviours
    SnapshotManager mockSnapshotManager = mock(SnapshotManager.class);
    when(mockSnapshotManager.getQueueOffsets()).thenReturn(queueOffsets);
    // mock SchemaManager
    SchemaManager mockSchemaManager = mock(SchemaManager.class);
    when(mockSchemaManager.getGraphDef()).thenReturn(graphDef);
    // mock SnapshotCache
    SnapshotCache mockSnapshotCache = mock(SnapshotCache.class);
    when(mockSnapshotCache.getSnapshotWithSchema()).thenReturn(snapshotWithSchema);
    // mock StoreBackupTaskSender behaviours
    StoreBackupTaskSender mockStoreBackupTaskSender = mock(StoreBackupTaskSender.class);
    doAnswer(invocation -> {
        int partitionOrStoreId = invocation.getArgument(0);
        int globalBackupId = invocation.getArgument(1);
        CompletionCallback<StoreBackupId> callback = invocation.getArgument(2);
        StoreBackupId storeBackupId = new StoreBackupId(globalBackupId);
        storeBackupId.addPartitionBackupId(partitionOrStoreId, globalBackupId);
        callback.onCompleted(storeBackupId);
        return null;
    }).when(mockStoreBackupTaskSender).createStoreBackup(anyInt(), anyInt(), any());
    doAnswer(invocation -> {
        CompletionCallback<Void> callback = invocation.getArgument(3);
        callback.onCompleted(null);
        return null;
    }).when(mockStoreBackupTaskSender).restoreFromStoreBackup(anyInt(), any(), anyString(), any());
    doAnswer(invocation -> {
        CompletionCallback<Void> callback = invocation.getArgument(2);
        callback.onCompleted(null);
        return null;
    }).when(mockStoreBackupTaskSender).verifyStoreBackup(anyInt(), any(), any());
    BackupManager backupManager = new BackupManager(configs, mockMetaService, mockMetaStore, mockSnapshotManager, mockSchemaManager, mockSnapshotCache, mockStoreBackupTaskSender);
    backupManager.start();
    verify(mockSnapshotManager).addListener(any());
    // create the first backup
    CountDownLatch updateBackupIdLatch1 = new CountDownLatch(1);
    CountDownLatch updateBackupInfoByCreation1Latch = new CountDownLatch(1);
    doAnswer(invocation -> {
        updateBackupIdLatch1.countDown();
        return null;
    }).when(mockMetaStore).write(GLOBAL_BACKUP_ID_PATH, objectMapper.writeValueAsBytes(1));
    doAnswer(invocation -> {
        byte[] backupInfoBytes = invocation.getArgument(1);
        List<BackupInfo> backupInfoList = objectMapper.readValue(backupInfoBytes, new TypeReference<List<BackupInfo>>() {
        });
        assertEquals(backupInfoList.size(), 1);
        assertEquals(backupInfoList.get(0), backupInfo1);
        updateBackupInfoByCreation1Latch.countDown();
        return null;
    }).when(mockMetaStore).write(BACKUP_INFO_PATH, objectMapper.writeValueAsBytes(Collections.singletonList(backupInfo1)));
    int backupId1 = backupManager.createNewBackup();
    assertEquals(backupId1, 1);
    assertTrue(updateBackupIdLatch1.await(5L, TimeUnit.SECONDS));
    assertTrue(updateBackupInfoByCreation1Latch.await(5L, TimeUnit.SECONDS));
    // create the second backup
    CountDownLatch updateBackupIdLatch2 = new CountDownLatch(1);
    CountDownLatch updateBackupInfoByCreation2Latch = new CountDownLatch(1);
    doAnswer(invocation -> {
        updateBackupIdLatch2.countDown();
        return null;
    }).when(mockMetaStore).write(GLOBAL_BACKUP_ID_PATH, objectMapper.writeValueAsBytes(2));
    doAnswer(invocation -> {
        byte[] backupInfoBytes = invocation.getArgument(1);
        List<BackupInfo> backupInfoList = objectMapper.readValue(backupInfoBytes, new TypeReference<List<BackupInfo>>() {
        });
        backupInfoList.sort(new Comparator<BackupInfo>() {

            @Override
            public int compare(BackupInfo o1, BackupInfo o2) {
                return o1.getGlobalBackupId() - o2.getGlobalBackupId();
            }
        });
        assertEquals(backupInfoList.size(), 2);
        assertEquals(backupInfoList.get(0), backupInfo1);
        assertEquals(backupInfoList.get(1), backupInfo2);
        updateBackupInfoByCreation2Latch.countDown();
        return null;
    }).when(mockMetaStore).write(BACKUP_INFO_PATH, objectMapper.writeValueAsBytes(Arrays.asList(backupInfo1, backupInfo2)));
    int backupId2 = backupManager.createNewBackup();
    assertEquals(backupId2, 2);
    assertTrue(updateBackupIdLatch2.await(5L, TimeUnit.SECONDS));
    assertTrue(updateBackupInfoByCreation2Latch.await(5L, TimeUnit.SECONDS));
    // get backup info list and check
    assertEquals(backupManager.getBackupInfoList().size(), 2);
    // verify backups
    try {
        backupManager.verifyBackup(backupId1);
        backupManager.verifyBackup(backupId2);
    } catch (Exception e) {
        fail("should not have thrown any exception during backup verification");
    }
    // restore from the second backup
    try {
        backupManager.restoreFromBackup(backupId2, "restore_meta", "restore_store");
        assertTrue(Files.exists(Paths.get("restore_meta", "query_snapshot_id")));
        assertTrue(Files.exists(Paths.get("restore_meta", "graph_def_proto_bytes")));
        assertTrue(Files.exists(Paths.get("restore_meta", "queue_offsets")));
    } catch (Exception e) {
        fail("should not have thrown any exception during backup restoring");
    } finally {
        FileUtils.deleteDirectory(new File("restore_meta"));
    }
    // purge 1 old backup
    CountDownLatch updateBackupInfoByPurgingLatch = new CountDownLatch(1);
    doAnswer(invocation -> {
        byte[] backupInfoBytes = invocation.getArgument(1);
        List<BackupInfo> backupInfoList = objectMapper.readValue(backupInfoBytes, new TypeReference<List<BackupInfo>>() {
        });
        assertEquals(backupInfoList.size(), 1);
        assertEquals(backupInfoList.get(0), backupInfo2);
        updateBackupInfoByPurgingLatch.countDown();
        return null;
    }).when(mockMetaStore).write(BACKUP_INFO_PATH, objectMapper.writeValueAsBytes(Collections.singletonList(backupInfo2)));
    backupManager.purgeOldBackups(1);
    assertTrue(updateBackupInfoByPurgingLatch.await(5L, TimeUnit.SECONDS));
    // get backup info list and check
    assertEquals(backupManager.getBackupInfoList().size(), 1);
    // delete the remaining backup '2'
    CountDownLatch updateBackupInfoByDeletionLatch = new CountDownLatch(1);
    doAnswer(invocation -> {
        byte[] backupInfoBytes = invocation.getArgument(1);
        List<BackupInfo> backupInfoList = objectMapper.readValue(backupInfoBytes, new TypeReference<List<BackupInfo>>() {
        });
        assertTrue(backupInfoList.isEmpty());
        updateBackupInfoByDeletionLatch.countDown();
        return null;
    }).when(mockMetaStore).write(BACKUP_INFO_PATH, objectMapper.writeValueAsBytes(new ArrayList<BackupInfo>()));
    backupManager.deleteBackup(2);
    assertTrue(updateBackupInfoByDeletionLatch.await(5L, TimeUnit.SECONDS));
    // get backup info list and check
    assertTrue(backupManager.getBackupInfoList().isEmpty());
    backupManager.stop();
    verify(mockSnapshotManager).removeListener(any());
}

Example 23 with Configs

use of com.alibaba.maxgraph.common.config.Configs in project GraphScope by alibaba.

the class LogRecyclerTest method testRecycler.

@Test
void testRecycler() throws IOException {
    Configs configs = Configs.newBuilder().put(CoordinatorConfig.LOG_RECYCLE_INTERVAL_SECOND.getKey(), "1").build();
    SnapshotManager mockSnapshotManager = mock(SnapshotManager.class);
    when(mockSnapshotManager.getQueueOffsets()).thenReturn(Arrays.asList(1L, 2L, 3L));
    LogService mockLogService = mock(LogService.class);
    LogRecycler logRecycler = new LogRecycler(configs, mockLogService, mockSnapshotManager);
    CountDownLatch latch1 = new CountDownLatch(1);
    CountDownLatch latch2 = new CountDownLatch(1);
    CountDownLatch latch3 = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        latch1.countDown();
        return null;
    }).when(mockLogService).deleteBeforeOffset(0, 1L);
    doAnswer(invocationOnMock -> {
        latch2.countDown();
        return null;
    }).when(mockLogService).deleteBeforeOffset(1, 2L);
    doAnswer(invocationOnMock -> {
        latch3.countDown();
        return null;
    }).when(mockLogService).deleteBeforeOffset(2, 3L);
    logRecycler.start();
    assertAll(() -> assertTrue(latch1.await(5L, TimeUnit.SECONDS)), () -> assertTrue(latch2.await(5L, TimeUnit.SECONDS)), () -> assertTrue(latch3.await(5L, TimeUnit.SECONDS)));
    logRecycler.stop();
}

Example 24 with Configs

use of com.alibaba.maxgraph.common.config.Configs in project GraphScope by alibaba.

the class BatchSenderTest method testSend.

@Test
void testSend() throws InterruptedException {
    Configs configs = Configs.newBuilder().put(CommonConfig.STORE_NODE_COUNT.getKey(), "1").build();
    MetaService mockMetaService = mock(MetaService.class);
    when(mockMetaService.getPartitionCount()).thenReturn(2);
    when(mockMetaService.getStoreIdByPartition(anyInt())).thenReturn(0);
    StoreWriter mockStoreWriter = mock(StoreWriter.class);
    String requestId = "test_batch_sender";
    int queueId = 0;
    long snapshotId = 10L;
    long offset = 50L;
    LabelId labelId = new LabelId(0);
    OperationBlob writeVertexBlob1 = new OverwriteVertexOperation(new VertexId(0L), labelId, Collections.EMPTY_MAP).toBlob();
    OperationBlob writeVertexBlob2 = new OverwriteVertexOperation(new VertexId(1L), labelId, Collections.EMPTY_MAP).toBlob();
    OperationBatch batch = OperationBatch.newBuilder().addOperationBlob(writeVertexBlob1).addOperationBlob(writeVertexBlob2).addOperationBlob(OperationBlob.MARKER_OPERATION_BLOB).build();
    CountDownLatch latch = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        StoreDataBatch storeBatch = invocationOnMock.getArgument(1);
        CompletionCallback callback = invocationOnMock.getArgument(2);
        assertAll(() -> assertEquals(storeBatch.getRequestId(), requestId), () -> assertEquals(storeBatch.getQueueId(), queueId), () -> assertEquals(storeBatch.getSnapshotId(), snapshotId), () -> assertEquals(storeBatch.getOffset(), offset), () -> assertEquals(storeBatch.getDataBatch().size(), 2));
        List<Map<Integer, OperationBatch>> dataBatch = storeBatch.getDataBatch();
        Map<Integer, OperationBatch> partitionToBatch = dataBatch.get(0);
        assertAll(() -> assertEquals(partitionToBatch.get(0).getOperationBlob(0), writeVertexBlob1), () -> assertEquals(partitionToBatch.get(1).getOperationBlob(0), writeVertexBlob2));
        assertEquals(dataBatch.get(1).get(-1).getOperationBlob(0), OperationBlob.MARKER_OPERATION_BLOB);
        callback.onCompleted(0);
        latch.countDown();
        return null;
    }).when(mockStoreWriter).write(anyInt(), any(), any());
    BatchSender batchSender = new BatchSender(configs, mockMetaService, mockStoreWriter, new MetricsCollector(configs));
    batchSender.start();
    batchSender.asyncSendWithRetry(requestId, queueId, snapshotId, offset, batch);
    assertTrue(latch.await(5L, TimeUnit.SECONDS));
    batchSender.stop();
}

Example 25 with Configs

use of com.alibaba.maxgraph.common.config.Configs in project GraphScope by alibaba.

the class IngestProcessorTest method testIngestProcessor.

@Test
void testIngestProcessor() throws IOException {
    long tailOffset = 50L;
    int queueId = 0;
    Configs configs = Configs.newBuilder().build();
    LogService mockLogService = mock(LogService.class);
    LogReader mockLogReader = mock(LogReader.class);
    when(mockLogService.createReader(queueId, tailOffset + 1)).thenReturn(mockLogReader);
    LogWriter mockLogWriter = mock(LogWriter.class);
    when(mockLogService.createWriter(queueId)).thenReturn(mockLogWriter);
    when(mockLogWriter.append(any())).thenReturn(tailOffset + 3);
    OperationBatch emptyBatch = OperationBatch.newBuilder().build();
    long readSnapshotId = 5L;
    ReadLogEntry readLogEntry1 = new ReadLogEntry(tailOffset + 1, readSnapshotId, emptyBatch);
    ReadLogEntry readLogEntry2 = new ReadLogEntry(tailOffset + 2, readSnapshotId, emptyBatch);
    when(mockLogReader.readNext()).thenReturn(readLogEntry1).thenReturn(readLogEntry2).thenReturn(null);
    BatchSender mockBatchSender = mock(BatchSender.class);
    AtomicLong ingestSnapshotId = new AtomicLong(10L);
    IngestProcessor ingestProcessor = new IngestProcessor(configs, mockLogService, mockBatchSender, queueId, ingestSnapshotId, new MetricsCollector(configs));
    ingestProcessor.setTailOffset(tailOffset);
    ingestProcessor.start();
    verify(mockBatchSender, timeout(5000L)).asyncSendWithRetry(eq(""), eq(queueId), eq(readSnapshotId), eq(tailOffset + 1), any());
    verify(mockBatchSender, timeout(5000L)).asyncSendWithRetry(eq(""), eq(queueId), eq(readSnapshotId), eq(tailOffset + 2), any());
    verify(mockLogReader).close();
    String requestId = "test_ingest_processor";
    IngestCallback mockIngestCallback = mock(IngestCallback.class);
    ingestProcessor.ingestBatch(requestId, emptyBatch, mockIngestCallback);
    verify(mockBatchSender, timeout(5000L)).asyncSendWithRetry(requestId, queueId, ingestSnapshotId.get(), tailOffset + 3, emptyBatch);
    verify(mockIngestCallback, timeout(5000L)).onSuccess(ingestSnapshotId.get());
    ingestProcessor.stop();
    verify(mockLogWriter, timeout(5000L)).close();
}