Examples with PagesSerde io.hetu.core.transport.execution.buffer.PagesSerde used on opensource projects

Example 1 with PagesSerde

use of io.hetu.core.transport.execution.buffer.PagesSerde in project hetu-core by openlookeng.

the class StreamingAggregationOperator method capture.

@Override
public Object capture(BlockEncodingSerdeProvider serdeProvider) {
    StreamingAggregationOperatorState myState = new StreamingAggregationOperatorState();
    myState.operatorContext = operatorContext.capture(serdeProvider);
    myState.systemMemoryContext = systemMemoryContext.getBytes();
    myState.userMemoryContext = userMemoryContext.getBytes();
    myState.aggregates = new Object[aggregates.size()];
    for (int i = 0; i < aggregates.size(); i++) {
        myState.aggregates[i] = aggregates.get(i).capture(serdeProvider);
    }
    myState.pageBuilder = pageBuilder.capture(serdeProvider);
    if (currentGroup != null) {
        SerializedPage serializedPage = ((PagesSerde) serdeProvider).serialize(currentGroup);
        myState.currentGroup = serializedPage.capture(serdeProvider);
    }
    myState.finishing = finishing;
    return myState;
}

Example 2 with PagesSerde

use of io.hetu.core.transport.execution.buffer.PagesSerde in project hetu-core by openlookeng.

the class TestPagesIndex method testEstimatedSizeSnapshot.

@Test
public void testEstimatedSizeSnapshot() {
    List<Type> types = ImmutableList.of(BIGINT, VARCHAR);
    PagesSerde serde = TestingPagesSerdeFactory.testingPagesSerde();
    Page input = somePage(types);
    PagesIndex pagesIndex = newPagesIndex(types, 30, false);
    long initialEstimatedSize = pagesIndex.getEstimatedSize().toBytes();
    assertTrue(initialEstimatedSize > 0, format("Initial estimated size must be positive, got %s", initialEstimatedSize));
    pagesIndex.addPage(input);
    long estimatedSizeWithOnePage = pagesIndex.getEstimatedSize().toBytes();
    assertTrue(estimatedSizeWithOnePage > initialEstimatedSize, "Estimated size should grow after adding a page");
    Object snapshot = pagesIndex.capture(serde);
    assertEquals(SnapshotTestUtil.toSimpleSnapshotMapping(snapshot), createExpectedMapping());
    pagesIndex.addPage(input);
    long estimatedSizeWithTwoPages = pagesIndex.getEstimatedSize().toBytes();
    assertEquals(estimatedSizeWithTwoPages, initialEstimatedSize + (estimatedSizeWithOnePage - initialEstimatedSize) * 2, "Estimated size should grow linearly as long as we don't pass expectedPositions");
    pagesIndex.compact();
    long estimatedSizeAfterCompact = pagesIndex.getEstimatedSize().toBytes();
    // We can expect compact to reduce size because VARCHAR sequence pages are compactable.
    assertTrue(estimatedSizeAfterCompact < estimatedSizeWithTwoPages, format("Compact should reduce (or retain) size, but changed from %s to %s", estimatedSizeWithTwoPages, estimatedSizeAfterCompact));
}

Example 3 with PagesSerde

use of io.hetu.core.transport.execution.buffer.PagesSerde in project hetu-core by openlookeng.

the class TestBinaryFileSpiller method testSpillerUnCommit.

private void testSpillerUnCommit(List<Type> types, List<Page>... spills) throws ExecutionException, InterruptedException, IOException {
    long spilledBytesBefore = spillerStats.getTotalSpilledBytes();
    long spilledBytes = 0;
    assertEquals(memoryContext.getBytes(), 0);
    List<Runnable> runners = new ArrayList<>();
    PagesSerde serde = TestingPagesSerdeFactory.testingPagesSerde();
    Spiller spiller = factory.create(TYPES, bytes -> {
    }, memoryContext);
    spilledBytes = doSpill(spiller, spilledBytes, runners, spills, 0);
    spillUploadPath.mkdirs();
    int counter = 1;
    int runCount = runners.size();
    List<Path> uploadedFile = new ArrayList<>();
    for (counter = 1; counter <= runCount; counter++) {
        runners.remove(0).run();
        assertEquals(spiller.getSpilledFilePaths().size(), counter);
        assertEquals(runners.size(), runCount - counter);
        Object snapshot = spiller.capture(serde);
        spiller.getSpilledFilePaths().stream().forEach(path -> {
            try {
                Files.copy(path, Paths.get(spillUploadPath.getPath(), path.getFileName().toString()), REPLACE_EXISTING);
                uploadedFile.add(Paths.get(spillUploadPath.getPath(), path.getFileName().toString()));
            } catch (IOException e) {
                e.printStackTrace();
            }
        });
        spiller.close();
        spiller = factory.create(TYPES, bytes -> {
        }, memoryContext);
        spiller.restore(snapshot, serde);
        uploadedFile.stream().forEach(path -> {
            try {
                Files.move(path, Paths.get(spillPath.getPath(), path.getFileName().toString()), REPLACE_EXISTING);
            } catch (IOException e) {
                e.printStackTrace();
            }
        });
        uploadedFile.clear();
        doSpill(spiller, spilledBytes, runners, spills, counter);
    }
    // assertEquals(spillerStats.getTotalSpilledBytes() - spilledBytesBefore, spilledBytes);
    // At this point, the buffers should still be accounted for in the memory context, because
    // the spiller (FileSingleStreamSpiller) doesn't release its memory reservation until it's closed.
    // assertEquals(memoryContext.getBytes(), spills.length * FileSingleStreamSpiller.BUFFER_SIZE);
    List<Iterator<Page>> actualSpills = spiller.getSpills();
    assertEquals(actualSpills.size(), spills.length);
    for (int i = 0; i < actualSpills.size(); i++) {
        List<Page> actualSpill = ImmutableList.copyOf(actualSpills.get(i));
        List<Page> expectedSpill = spills[i];
        assertEquals(actualSpill.size(), expectedSpill.size());
        for (int j = 0; j < actualSpill.size(); j++) {
            assertPageEquals(types, actualSpill.get(j), expectedSpill.get(j));
        }
    }
    spiller.close();
    assertEquals(memoryContext.getBytes(), 0);
}

Example 4 with PagesSerde

use of io.hetu.core.transport.execution.buffer.PagesSerde in project hetu-core by openlookeng.

the class TestSpillCipherPagesSerde method test.

@Test
public void test() {
    SpillCipher cipher = new AesSpillCipher();
    PagesSerde serde = TESTING_SERDE_FACTORY.createPagesSerdeForSpill(Optional.of(cipher), false, false);
    List<Type> types = ImmutableList.of(VARCHAR);
    Page emptyPage = new Page(VARCHAR.createBlockBuilder(null, 0).build());
    assertPageEquals(types, serde.deserialize(serde.serialize(emptyPage)), emptyPage);
    BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 2);
    VARCHAR.writeString(blockBuilder, "hello");
    VARCHAR.writeString(blockBuilder, "world");
    Page helloWorldPage = new Page(blockBuilder.build());
    SerializedPage serialized = serde.serialize(helloWorldPage);
    assertPageEquals(types, serde.deserialize(serialized), helloWorldPage);
    assertTrue(serialized.isEncrypted(), "page should be encrypted");
    cipher.close();
    assertFailure(() -> serde.serialize(helloWorldPage), "Spill cipher already closed");
    assertFailure(() -> serde.deserialize(serialized), "Spill cipher already closed");
}

Example 5 with PagesSerde

use of io.hetu.core.transport.execution.buffer.PagesSerde in project hetu-core by openlookeng.

the class TestMarkerPage method testStaticSerde.

@Test
public void testStaticSerde() {
    PagesSerde serde = TESTING_SERDE_FACTORY.createPagesSerde();
    MarkerPage marker1 = MarkerPage.snapshotPage(1);
    SerializedPage serializedPage = serde.serialize(marker1);
    Page deserializedPage = serde.deserialize(serializedPage);
    Assert.assertTrue(deserializedPage instanceof MarkerPage);
    marker1 = (MarkerPage) deserializedPage;
    Assert.assertEquals(marker1.getSnapshotId(), 1);
    Assert.assertFalse(marker1.isResuming());
    MarkerPage resume1 = MarkerPage.resumePage(1);
    serializedPage = serde.serialize(resume1);
    deserializedPage = serde.deserialize(serializedPage);
    Assert.assertTrue(deserializedPage instanceof MarkerPage);
    resume1 = (MarkerPage) deserializedPage;
    Assert.assertEquals(resume1.getSnapshotId(), 1);
    Assert.assertTrue(resume1.isResuming());
    Page regular = new Page(1);
    serializedPage = serde.serialize(regular);
    deserializedPage = serde.deserialize(serializedPage);
    Assert.assertFalse(deserializedPage instanceof MarkerPage);
}