Examples with StoreDataBatch com.alibaba.graphscope.groot.operation.StoreDataBatch used on opensource projects

Example 1 with StoreDataBatch

use of com.alibaba.graphscope.groot.operation.StoreDataBatch in project GraphScope by alibaba.

the class SnapshotSortQueue method poll.

public StoreDataBatch poll() throws InterruptedException {
    if (this.currentPollSnapshotId == -1L) {
        // We need to wait for all queues each has at least 1 entry to decide initial
        // pollSnapshotId
        long minSnapshotId = Long.MAX_VALUE;
        for (int i = 0; i < this.innerQueues.size(); i++) {
            StoreDataBatch entry = this.queueHeads.get(i);
            if (entry == null) {
                entry = this.innerQueues.get(i).poll(this.queueWaitMs, TimeUnit.MILLISECONDS);
                if (entry == null) {
                    return null;
                }
                this.queueHeads.set(i, entry);
            }
            long entrySnapshotId = entry.getSnapshotId();
            if (entrySnapshotId < minSnapshotId) {
                minSnapshotId = entrySnapshotId;
            }
        }
        this.currentPollSnapshotId = minSnapshotId;
        logger.info("currentPollSnapshotId initialize to [" + this.currentPollSnapshotId + "]");
    }
    while (true) {
        StoreDataBatch entry = this.queueHeads.get(this.currentPollQueueIdx);
        this.queueHeads.set(this.currentPollQueueIdx, null);
        if (entry == null) {
            entry = this.innerQueues.get(this.currentPollQueueIdx).poll(this.queueWaitMs, TimeUnit.MILLISECONDS);
            if (entry == null) {
                return null;
            }
        }
        long snapshotId = entry.getSnapshotId();
        if (snapshotId == this.currentPollSnapshotId) {
            this.size.decrementAndGet();
            return entry;
        }
        if (snapshotId > this.currentPollSnapshotId) {
            this.queueHeads.set(this.currentPollQueueIdx, entry);
            this.currentPollQueueIdx--;
            if (this.currentPollQueueIdx == -1) {
                this.currentPollQueueIdx = this.queueCount - 1;
                this.currentPollSnapshotId++;
            }
        } else {
            logger.warn("Illegal entry polled from queue [" + this.currentPollQueueIdx + "]. entrySnapshotId [" + snapshotId + "] < currentSnapshotId [" + this.currentPollSnapshotId + "]. Ignored entry.");
        }
    }
}

Example 2 with StoreDataBatch

use of com.alibaba.graphscope.groot.operation.StoreDataBatch in project GraphScope by alibaba.

the class SnapshotSortQueueTest method testQueue.

@Test
void testQueue() throws InterruptedException {
    Configs configs = Configs.newBuilder().build();
    MetaService mockMetaService = mock(MetaService.class);
    when(mockMetaService.getQueueCount()).thenReturn(2);
    SnapshotSortQueue snapshotSortQueue = new SnapshotSortQueue(configs, mockMetaService);
    /**
     * Q1: 4, 5, 7, 7 Q0: 4, 5, 6, 7
     *
     * <p>(Q1, 4) (Q0, 4) (Q1, 5) (Q0, 5) (Q0, 6) (Q1, 7) (Q1, 7) (Q0, 7)
     */
    snapshotSortQueue.offerQueue(1, StoreDataBatch.newBuilder().snapshotId(4L).queueId(1).build());
    snapshotSortQueue.offerQueue(1, StoreDataBatch.newBuilder().snapshotId(5L).queueId(1).build());
    snapshotSortQueue.offerQueue(1, StoreDataBatch.newBuilder().snapshotId(7L).queueId(1).build());
    snapshotSortQueue.offerQueue(1, StoreDataBatch.newBuilder().snapshotId(7L).queueId(1).build());
    // For end snapshot 7
    snapshotSortQueue.offerQueue(1, StoreDataBatch.newBuilder().snapshotId(100L).queueId(1).build());
    snapshotSortQueue.offerQueue(0, StoreDataBatch.newBuilder().snapshotId(4L).queueId(0).build());
    snapshotSortQueue.offerQueue(0, StoreDataBatch.newBuilder().snapshotId(5L).queueId(0).build());
    snapshotSortQueue.offerQueue(0, StoreDataBatch.newBuilder().snapshotId(6L).queueId(0).build());
    snapshotSortQueue.offerQueue(0, StoreDataBatch.newBuilder().snapshotId(7L).queueId(0).build());
    StoreDataBatch entry = snapshotSortQueue.poll();
    assertEquals(entry.getQueueId(), 1);
    assertEquals(entry.getSnapshotId(), 4L);
    entry = snapshotSortQueue.poll();
    assertEquals(entry.getQueueId(), 0);
    assertEquals(entry.getSnapshotId(), 4L);
    entry = snapshotSortQueue.poll();
    assertEquals(entry.getQueueId(), 1);
    assertEquals(entry.getSnapshotId(), 5L);
    entry = snapshotSortQueue.poll();
    assertEquals(entry.getQueueId(), 0);
    assertEquals(entry.getSnapshotId(), 5L);
    entry = snapshotSortQueue.poll();
    assertEquals(entry.getQueueId(), 0);
    assertEquals(entry.getSnapshotId(), 6L);
    entry = snapshotSortQueue.poll();
    assertEquals(entry.getQueueId(), 1);
    assertEquals(entry.getSnapshotId(), 7L);
    entry = snapshotSortQueue.poll();
    assertEquals(entry.getQueueId(), 1);
    assertEquals(entry.getSnapshotId(), 7L);
    entry = snapshotSortQueue.poll();
    assertEquals(entry.getQueueId(), 0);
    assertEquals(entry.getSnapshotId(), 7L);
}

Example 3 with StoreDataBatch

use of com.alibaba.graphscope.groot.operation.StoreDataBatch in project GraphScope by alibaba.

the class StoreServiceTest method testStoreService.

@Test
void testStoreService() throws IOException, InterruptedException, ExecutionException {
    Configs configs = Configs.newBuilder().put(CommonConfig.NODE_IDX.getKey(), "0").build();
    MetaService mockMetaService = mock(MetaService.class);
    when(mockMetaService.getPartitionsByStoreId(0)).thenReturn(Arrays.asList(0));
    StoreService spyStoreService = spy(new StoreService(configs, mockMetaService, new MetricsCollector(configs)));
    GraphPartition mockGraphPartition = mock(GraphPartition.class);
    when(mockGraphPartition.recover()).thenReturn(10L);
    doReturn(mockGraphPartition).when(spyStoreService).makeGraphPartition(any(), eq(0));
    spyStoreService.start();
    assertEquals(spyStoreService.recover(), 10L);
    StoreDataBatch storeDataBatch = StoreDataBatch.newBuilder().snapshotId(20L).addOperation(0, OperationBlob.MARKER_OPERATION_BLOB).build();
    spyStoreService.batchWrite(storeDataBatch);
    verify(mockGraphPartition, timeout(100L)).writeBatch(20L, OperationBatch.newBuilder().addOperationBlob(OperationBlob.MARKER_OPERATION_BLOB).build());
    spyStoreService.stop();
    verify(mockGraphPartition).close();
}

Example 4 with StoreDataBatch

use of com.alibaba.graphscope.groot.operation.StoreDataBatch in project GraphScope by alibaba.

the class WriterAgentTest method testWriterAgent.

@Test
void testWriterAgent() throws InterruptedException, ExecutionException {
    Configs configs = Configs.newBuilder().put(CommonConfig.NODE_IDX.getKey(), "0").put(StoreConfig.STORE_COMMIT_INTERVAL_MS.getKey(), "10").build();
    StoreService mockStoreService = mock(StoreService.class);
    MetaService mockMetaService = mock(MetaService.class);
    when(mockMetaService.getQueueCount()).thenReturn(1);
    SnapshotCommitter mockSnapshotCommitter = mock(SnapshotCommitter.class);
    WriterAgent writerAgent = new WriterAgent(configs, mockStoreService, mockMetaService, mockSnapshotCommitter, new MetricsCollector(configs));
    writerAgent.init(0L);
    writerAgent.start();
    StoreDataBatch storeDataBatch = StoreDataBatch.newBuilder().snapshotId(2L).queueId(0).offset(10L).build();
    writerAgent.writeStore(storeDataBatch);
    verify(mockStoreService, timeout(5000L).times(1)).batchWrite(storeDataBatch);
    verify(mockSnapshotCommitter, timeout(5000L).times(1)).commitSnapshotId(0, 1L, 0L, Collections.singletonList(10L));
    writerAgent.stop();
}

Example 5 with StoreDataBatch

use of com.alibaba.graphscope.groot.operation.StoreDataBatch in project GraphScope by alibaba.

the class WriterAgent method writeStore2.

public boolean writeStore2(List<StoreDataBatch> storeDataBatches) throws InterruptedException {
    long beforeOfferTime = System.nanoTime();
    for (StoreDataBatch storeDataBatch : storeDataBatches) {
        int queueId = storeDataBatch.getQueueId();
        if (!this.bufferQueue.offerQueue(queueId, storeDataBatch)) {
            return false;
        }
    }
    long afterOfferTime = System.nanoTime();
    this.bufferWritePerSecondMetric.add(afterOfferTime - beforeOfferTime);
    return true;
}