use of com.alibaba.maxgraph.common.config.Configs in project GraphScope by alibaba.
the class SnapshotSortQueueTest method testQueue.
@Test
void testQueue() throws InterruptedException {
Configs configs = Configs.newBuilder().build();
MetaService mockMetaService = mock(MetaService.class);
when(mockMetaService.getQueueCount()).thenReturn(2);
SnapshotSortQueue snapshotSortQueue = new SnapshotSortQueue(configs, mockMetaService);
/**
* Q1: 4, 5, 7, 7 Q0: 4, 5, 6, 7
*
* <p>(Q1, 4) (Q0, 4) (Q1, 5) (Q0, 5) (Q0, 6) (Q1, 7) (Q1, 7) (Q0, 7)
*/
snapshotSortQueue.offerQueue(1, StoreDataBatch.newBuilder().snapshotId(4L).queueId(1).build());
snapshotSortQueue.offerQueue(1, StoreDataBatch.newBuilder().snapshotId(5L).queueId(1).build());
snapshotSortQueue.offerQueue(1, StoreDataBatch.newBuilder().snapshotId(7L).queueId(1).build());
snapshotSortQueue.offerQueue(1, StoreDataBatch.newBuilder().snapshotId(7L).queueId(1).build());
// For end snapshot 7
snapshotSortQueue.offerQueue(1, StoreDataBatch.newBuilder().snapshotId(100L).queueId(1).build());
snapshotSortQueue.offerQueue(0, StoreDataBatch.newBuilder().snapshotId(4L).queueId(0).build());
snapshotSortQueue.offerQueue(0, StoreDataBatch.newBuilder().snapshotId(5L).queueId(0).build());
snapshotSortQueue.offerQueue(0, StoreDataBatch.newBuilder().snapshotId(6L).queueId(0).build());
snapshotSortQueue.offerQueue(0, StoreDataBatch.newBuilder().snapshotId(7L).queueId(0).build());
StoreDataBatch entry = snapshotSortQueue.poll();
assertEquals(entry.getQueueId(), 1);
assertEquals(entry.getSnapshotId(), 4L);
entry = snapshotSortQueue.poll();
assertEquals(entry.getQueueId(), 0);
assertEquals(entry.getSnapshotId(), 4L);
entry = snapshotSortQueue.poll();
assertEquals(entry.getQueueId(), 1);
assertEquals(entry.getSnapshotId(), 5L);
entry = snapshotSortQueue.poll();
assertEquals(entry.getQueueId(), 0);
assertEquals(entry.getSnapshotId(), 5L);
entry = snapshotSortQueue.poll();
assertEquals(entry.getQueueId(), 0);
assertEquals(entry.getSnapshotId(), 6L);
entry = snapshotSortQueue.poll();
assertEquals(entry.getQueueId(), 1);
assertEquals(entry.getSnapshotId(), 7L);
entry = snapshotSortQueue.poll();
assertEquals(entry.getQueueId(), 1);
assertEquals(entry.getSnapshotId(), 7L);
entry = snapshotSortQueue.poll();
assertEquals(entry.getQueueId(), 0);
assertEquals(entry.getSnapshotId(), 7L);
}
use of com.alibaba.maxgraph.common.config.Configs in project GraphScope by alibaba.
the class StoreServiceTest method testStoreService.
@Test
void testStoreService() throws IOException, InterruptedException, ExecutionException {
Configs configs = Configs.newBuilder().put(CommonConfig.NODE_IDX.getKey(), "0").build();
MetaService mockMetaService = mock(MetaService.class);
when(mockMetaService.getPartitionsByStoreId(0)).thenReturn(Arrays.asList(0));
StoreService spyStoreService = spy(new StoreService(configs, mockMetaService, new MetricsCollector(configs)));
GraphPartition mockGraphPartition = mock(GraphPartition.class);
when(mockGraphPartition.recover()).thenReturn(10L);
doReturn(mockGraphPartition).when(spyStoreService).makeGraphPartition(any(), eq(0));
spyStoreService.start();
assertEquals(spyStoreService.recover(), 10L);
StoreDataBatch storeDataBatch = StoreDataBatch.newBuilder().snapshotId(20L).addOperation(0, OperationBlob.MARKER_OPERATION_BLOB).build();
spyStoreService.batchWrite(storeDataBatch);
verify(mockGraphPartition, timeout(100L)).writeBatch(20L, OperationBatch.newBuilder().addOperationBlob(OperationBlob.MARKER_OPERATION_BLOB).build());
spyStoreService.stop();
verify(mockGraphPartition).close();
}
use of com.alibaba.maxgraph.common.config.Configs in project GraphScope by alibaba.
the class WriterAgentTest method testWriterAgent.
@Test
void testWriterAgent() throws InterruptedException, ExecutionException {
Configs configs = Configs.newBuilder().put(CommonConfig.NODE_IDX.getKey(), "0").put(StoreConfig.STORE_COMMIT_INTERVAL_MS.getKey(), "10").build();
StoreService mockStoreService = mock(StoreService.class);
MetaService mockMetaService = mock(MetaService.class);
when(mockMetaService.getQueueCount()).thenReturn(1);
SnapshotCommitter mockSnapshotCommitter = mock(SnapshotCommitter.class);
WriterAgent writerAgent = new WriterAgent(configs, mockStoreService, mockMetaService, mockSnapshotCommitter, new MetricsCollector(configs));
writerAgent.init(0L);
writerAgent.start();
StoreDataBatch storeDataBatch = StoreDataBatch.newBuilder().snapshotId(2L).queueId(0).offset(10L).build();
writerAgent.writeStore(storeDataBatch);
verify(mockStoreService, timeout(5000L).times(1)).batchWrite(storeDataBatch);
verify(mockSnapshotCommitter, timeout(5000L).times(1)).commitSnapshotId(0, 1L, 0L, Collections.singletonList(10L));
writerAgent.stop();
}
use of com.alibaba.maxgraph.common.config.Configs in project GraphScope by alibaba.
the class ExecutorDiscoveryManagerTest method testExecutorServerStart.
@Test
void testExecutorServerStart() throws Exception {
try (TestingServer testingServer = new TestingServer(-1)) {
int zkPort = testingServer.getPort();
Configs configs = Configs.newBuilder().put(ZkConfig.ZK_CONNECT_STRING.getKey(), "localhost:" + zkPort).put(ZkConfig.ZK_BASE_PATH.getKey(), "test_discovery").build();
CuratorFramework curator = CuratorUtils.makeCurator(configs);
curator.start();
LocalNodeProvider engineServerProvider = new LocalNodeProvider(RoleType.EXECUTOR_ENGINE, configs);
NodeDiscovery engineServerDiscovery = new ZkDiscovery(configs, engineServerProvider, curator);
LocalNodeProvider storeQueryProvider = new LocalNodeProvider(RoleType.EXECUTOR_GRAPH, configs);
NodeDiscovery storeQueryDiscovery = new ZkDiscovery(configs, storeQueryProvider, curator);
LocalNodeProvider queryExecuteProvider = new LocalNodeProvider(RoleType.EXECUTOR_QUERY, configs);
NodeDiscovery queryExecuteDiscovery = new ZkDiscovery(configs, queryExecuteProvider, curator);
LocalNodeProvider queryManageProvider = new LocalNodeProvider(RoleType.EXECUTOR_MANAGE, configs);
NodeDiscovery queryManageDiscovery = new ZkDiscovery(configs, queryManageProvider, curator);
ExecutorDiscoveryManager executorDiscoveryManager = new ExecutorDiscoveryManager(engineServerProvider, engineServerDiscovery, storeQueryProvider, storeQueryDiscovery, queryExecuteProvider, queryExecuteDiscovery, queryManageProvider, queryManageDiscovery);
executorDiscoveryManager.getEngineServerProvider().apply(123);
executorDiscoveryManager.getEngineServerDiscovery().start();
executorDiscoveryManager.getQueryExecuteProvider().apply(1234);
executorDiscoveryManager.getQueryExecuteDiscovery().start();
}
}
use of com.alibaba.maxgraph.common.config.Configs in project GraphScope by alibaba.
the class KafkaWalTest method testLogService.
@Test
void testLogService() throws IOException {
Configs configs = Configs.newBuilder().put(KafkaConfig.KAFKA_SERVERS.getKey(), sharedKafkaTestResource.getKafkaConnectString()).put(KafkaConfig.KAKFA_TOPIC.getKey(), "test_logservice").put(CommonConfig.INGESTOR_QUEUE_COUNT.getKey(), "1").build();
LogService logService = new KafkaLogService(configs);
logService.init();
int queueId = 0;
long snapshotId = 1L;
LogWriter writer = logService.createWriter(queueId);
LogEntry logEntry = new LogEntry(snapshotId, OperationBatch.newBuilder().addOperationBlob(OperationBlob.MARKER_OPERATION_BLOB).build());
assertEquals(writer.append(logEntry), 0);
LogReader reader = logService.createReader(queueId, 0);
ReadLogEntry readLogEntry = reader.readNext();
reader.close();
assertAll(() -> assertEquals(readLogEntry.getOffset(), 0), () -> assertEquals(readLogEntry.getLogEntry().getSnapshotId(), snapshotId));
OperationBatch operationBatch = readLogEntry.getLogEntry().getOperationBatch();
assertEquals(operationBatch.getOperationCount(), 1);
assertEquals(operationBatch.getOperationBlob(0), OperationBlob.MARKER_OPERATION_BLOB);
assertEquals(writer.append(logEntry), 1);
assertEquals(writer.append(logEntry), 2);
assertEquals(writer.append(logEntry), 3);
LogReader readerTail = logService.createReader(queueId, 4);
assertNull(readerTail.readNext());
readerTail.close();
assertThrows(IllegalArgumentException.class, () -> logService.createReader(queueId, 5));
logService.deleteBeforeOffset(queueId, 2);
assertThrows(IllegalArgumentException.class, () -> logService.createReader(queueId, 1));
writer.close();
logService.destroy();
}
Aggregations