Examples with SerializedPage io.hetu.core.transport.execution.buffer.SerializedPage used on opensource projects

Example 1 with SerializedPage

use of io.hetu.core.transport.execution.buffer.SerializedPage in project hetu-core by openlookeng.

the class StreamingAggregationOperator method capture.

@Override
public Object capture(BlockEncodingSerdeProvider serdeProvider) {
    StreamingAggregationOperatorState myState = new StreamingAggregationOperatorState();
    myState.operatorContext = operatorContext.capture(serdeProvider);
    myState.systemMemoryContext = systemMemoryContext.getBytes();
    myState.userMemoryContext = userMemoryContext.getBytes();
    myState.aggregates = new Object[aggregates.size()];
    for (int i = 0; i < aggregates.size(); i++) {
        myState.aggregates[i] = aggregates.get(i).capture(serdeProvider);
    }
    myState.pageBuilder = pageBuilder.capture(serdeProvider);
    if (currentGroup != null) {
        SerializedPage serializedPage = ((PagesSerde) serdeProvider).serialize(currentGroup);
        myState.currentGroup = serializedPage.capture(serdeProvider);
    }
    myState.finishing = finishing;
    return myState;
}

Example 2 with SerializedPage

use of io.hetu.core.transport.execution.buffer.SerializedPage in project hetu-core by openlookeng.

the class TestPartitionedOutputOperator method testPartitionedOutputOperatorSnapshot.

@Test
public void testPartitionedOutputOperatorSnapshot() throws Exception {
    SnapshotUtils snapshotUtils = mock(SnapshotUtils.class);
    PartitionedOutputBuffer buffer = mock(PartitionedOutputBuffer.class);
    PartitionedOutputOperator operator = createPartitionedOutputOperator(snapshotUtils, buffer);
    operator.getOperatorContext().getDriverContext().getPipelineContext().getTaskContext().getSnapshotManager().setTotalComponents(1);
    List<Page> input = rowPagesBuilder(BIGINT).addSequencePage(1, 1).addSequencePage(2, 4).build();
    MarkerPage marker = MarkerPage.snapshotPage(1);
    MarkerPage marker2 = MarkerPage.snapshotPage(2);
    MarkerPage marker3 = MarkerPage.snapshotPage(3);
    MarkerPage resume = MarkerPage.resumePage(1);
    // Add first page, then capture and compare, then add second page, then restore, then compare, then add second page, then finish, then compare
    operator.addInput(input.get(0));
    operator.addInput(marker);
    ArgumentCaptor<Object> stateArgument = ArgumentCaptor.forClass(Object.class);
    verify(snapshotUtils, times(1)).storeState(anyObject(), stateArgument.capture(), anyObject());
    Object snapshot = stateArgument.getValue();
    when(snapshotUtils.loadState(anyObject(), anyObject())).thenReturn(Optional.of(snapshot));
    operator.addInput(input.get(1));
    operator.addInput(resume);
    operator.addInput(marker2);
    verify(snapshotUtils, times(2)).storeState(anyObject(), stateArgument.capture(), anyObject());
    snapshot = stateArgument.getValue();
    Object snapshotEntry = ((Map<String, Object>) snapshot).get("query/2/1/1/0/0/0");
    assertEquals(SnapshotTestUtil.toFullSnapshotMapping(snapshotEntry), createExpectedMappingBeforeFinish());
    operator.addInput(input.get(1));
    operator.finish();
    operator.addInput(marker3);
    verify(snapshotUtils, times(3)).storeState(anyObject(), stateArgument.capture(), anyObject());
    snapshot = stateArgument.getValue();
    snapshotEntry = ((Map<String, Object>) snapshot).get("query/3/1/1/0/0/0");
    assertEquals(SnapshotTestUtil.toFullSnapshotMapping(snapshotEntry), createExpectedMappingAfterFinish());
    ArgumentCaptor<List> pagesArgument = ArgumentCaptor.forClass(List.class);
    verify(buffer, times(9)).enqueue(anyInt(), pagesArgument.capture(), anyString());
    List<List> pages = pagesArgument.getAllValues();
    // 9 pages:
    // 1 (page 1 partitioned)
    // 1 (marker 1)
    // 2 (page 2 before resume)
    // 1 (resume marker)
    // 1 (marker 2)
    // 2 (page 2 after resume)
    // 1 (marker 3)
    assertEquals(pages.size(), 9);
    assertTrue(((SerializedPage) pages.get(1).get(0)).isMarkerPage());
    assertTrue(((SerializedPage) pages.get(4).get(0)).isMarkerPage());
    assertTrue(((SerializedPage) pages.get(5).get(0)).isMarkerPage());
    assertTrue(((SerializedPage) pages.get(8).get(0)).isMarkerPage());
}

Example 3 with SerializedPage

use of io.hetu.core.transport.execution.buffer.SerializedPage in project hetu-core by openlookeng.

the class PartitionedOutputBuffer method enqueue.

@Override
public void enqueue(int partitionNumber, List<SerializedPage> pages, String origin) {
    requireNonNull(pages, "pages is null");
    // this can happen with a limit query
    if (!state.get().canAddPages()) {
        return;
    }
    if (!isSnapshotEnabled) {
        doEnqueue(partitionNumber, pages);
        return;
    }
    // but still arrives at the buffer *before* the marker. These pages are potentially used twice, if we resume from this marker.
    if (pages.size() == 1 && pages.get(0).isMarkerPage()) {
        // Broadcast marker to all clients
        for (int i = 0; i < partitions.size(); i++) {
            MultiInputSnapshotState snapshotState = snapshotStates.get(i);
            synchronized (snapshotState) {
                // All marker related processing is handled by this utility method
                List<SerializedPage> processedPages = snapshotState.processSerializedPages(pages, origin);
                doEnqueue(i, processedPages);
            }
        }
    } else {
        MultiInputSnapshotState snapshotState = snapshotStates.get(partitionNumber);
        synchronized (snapshotState) {
            List<SerializedPage> processedPages = snapshotState.processSerializedPages(pages, origin);
            doEnqueue(partitionNumber, processedPages);
        }
    }
}

Example 4 with SerializedPage

use of io.hetu.core.transport.execution.buffer.SerializedPage in project hetu-core by openlookeng.

the class TestSpillCipherPagesSerde method test.

@Test
public void test() {
    SpillCipher cipher = new AesSpillCipher();
    PagesSerde serde = TESTING_SERDE_FACTORY.createPagesSerdeForSpill(Optional.of(cipher), false, false);
    List<Type> types = ImmutableList.of(VARCHAR);
    Page emptyPage = new Page(VARCHAR.createBlockBuilder(null, 0).build());
    assertPageEquals(types, serde.deserialize(serde.serialize(emptyPage)), emptyPage);
    BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 2);
    VARCHAR.writeString(blockBuilder, "hello");
    VARCHAR.writeString(blockBuilder, "world");
    Page helloWorldPage = new Page(blockBuilder.build());
    SerializedPage serialized = serde.serialize(helloWorldPage);
    assertPageEquals(types, serde.deserialize(serialized), helloWorldPage);
    assertTrue(serialized.isEncrypted(), "page should be encrypted");
    cipher.close();
    assertFailure(() -> serde.serialize(helloWorldPage), "Spill cipher already closed");
    assertFailure(() -> serde.deserialize(serialized), "Spill cipher already closed");
}

Example 5 with SerializedPage

use of io.hetu.core.transport.execution.buffer.SerializedPage in project hetu-core by openlookeng.

the class TestMarkerPage method testSerde.

@Test
public void testSerde() {
    MarkerPage page = new MarkerPage(100, true);
    SerializedPage spage = SerializedPage.forMarker(page);
    page = spage.toMarker();
    Assert.assertEquals(page.getSnapshotId(), 100);
    Assert.assertTrue(page.isResuming());
}