Examples with TestingBlockEncodingSerde io.trino.spi.block.TestingBlockEncodingSerde used on opensource projects

Example 6 with TestingBlockEncodingSerde

use of io.trino.spi.block.TestingBlockEncodingSerde in project trino by trinodb.

the class TestFileSingleStreamSpiller method assertSpill.

private void assertSpill(boolean compression, boolean encryption) throws Exception {
    FileSingleStreamSpillerFactory spillerFactory = new FileSingleStreamSpillerFactory(// executor won't be closed, because we don't call destroy() on the spiller factory
    executor, new TestingBlockEncodingSerde(), new SpillerStats(), ImmutableList.of(spillPath.toPath()), 1.0, compression, encryption);
    LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext("test");
    SingleStreamSpiller singleStreamSpiller = spillerFactory.create(TYPES, bytes -> {
    }, memoryContext);
    assertTrue(singleStreamSpiller instanceof FileSingleStreamSpiller);
    FileSingleStreamSpiller spiller = (FileSingleStreamSpiller) singleStreamSpiller;
    Page page = buildPage();
    // The spillers will reserve memory in their constructors
    assertEquals(memoryContext.getBytes(), 4096);
    spiller.spill(page).get();
    spiller.spill(Iterators.forArray(page, page, page)).get();
    assertEquals(listFiles(spillPath.toPath()).size(), 1);
    // Assert the spill codec flags match the expected configuration
    try (InputStream is = newInputStream(listFiles(spillPath.toPath()).get(0))) {
        Iterator<Slice> serializedPages = PagesSerdeUtil.readSerializedPages(is);
        assertTrue(serializedPages.hasNext(), "at least one page should be successfully read back");
        Slice serializedPage = serializedPages.next();
        assertEquals(isSerializedPageCompressed(serializedPage), compression);
        assertEquals(isSerializedPageEncrypted(serializedPage), encryption);
    }
    // The spillers release their memory reservations when they are closed, therefore at this point
    // they will have non-zero memory reservation.
    // assertEquals(memoryContext.getBytes(), 0);
    Iterator<Page> spilledPagesIterator = spiller.getSpilledPages();
    assertEquals(memoryContext.getBytes(), FileSingleStreamSpiller.BUFFER_SIZE);
    ImmutableList<Page> spilledPages = ImmutableList.copyOf(spilledPagesIterator);
    // The spillers release their memory reservations when they are closed, therefore at this point
    // they will have non-zero memory reservation.
    // assertEquals(memoryContext.getBytes(), 0);
    assertEquals(4, spilledPages.size());
    for (int i = 0; i < 4; ++i) {
        PageAssertions.assertPageEquals(TYPES, page, spilledPages.get(i));
    }
    spiller.close();
    assertEquals(listFiles(spillPath.toPath()).size(), 0);
    assertEquals(memoryContext.getBytes(), 0);
}

Example 7 with TestingBlockEncodingSerde

use of io.trino.spi.block.TestingBlockEncodingSerde in project trino by trinodb.

the class TestGenericPartitioningSpiller method setUp.

@BeforeClass
public void setUp() throws Exception {
    tempDirectory = createTempDirectory(getClass().getSimpleName());
    FeaturesConfig featuresConfig = new FeaturesConfig();
    featuresConfig.setSpillerSpillPaths(tempDirectory.toString());
    featuresConfig.setSpillerThreads(8);
    featuresConfig.setSpillMaxUsedSpaceThreshold(1.0);
    SingleStreamSpillerFactory singleStreamSpillerFactory = new FileSingleStreamSpillerFactory(new TestingBlockEncodingSerde(), new SpillerStats(), featuresConfig, new NodeSpillConfig());
    factory = new GenericPartitioningSpillerFactory(singleStreamSpillerFactory);
    scheduledExecutor = newSingleThreadScheduledExecutor();
}

Example 8 with TestingBlockEncodingSerde

use of io.trino.spi.block.TestingBlockEncodingSerde in project trino by trinodb.

the class TestBinaryFileSpiller method setUp.

@BeforeMethod
public void setUp() {
    spillerStats = new SpillerStats();
    FeaturesConfig featuresConfig = new FeaturesConfig();
    featuresConfig.setSpillerSpillPaths(spillPath.getAbsolutePath());
    featuresConfig.setSpillMaxUsedSpaceThreshold(1.0);
    NodeSpillConfig nodeSpillConfig = new NodeSpillConfig();
    BlockEncodingSerde blockEncodingSerde = new TestingBlockEncodingSerde();
    singleStreamSpillerFactory = new FileSingleStreamSpillerFactory(blockEncodingSerde, spillerStats, featuresConfig, nodeSpillConfig);
    factory = new GenericSpillerFactory(singleStreamSpillerFactory);
    PagesSerdeFactory pagesSerdeFactory = new PagesSerdeFactory(blockEncodingSerde, nodeSpillConfig.isSpillCompressionEnabled());
    pagesSerde = pagesSerdeFactory.createPagesSerde();
    memoryContext = newSimpleAggregatedMemoryContext();
}

Example 9 with TestingBlockEncodingSerde

use of io.trino.spi.block.TestingBlockEncodingSerde in project trino by trinodb.

the class TestSortedRangeSet method testJsonSerialization.

@Test
public void testJsonSerialization() throws Exception {
    TestingTypeManager typeManager = new TestingTypeManager();
    TestingBlockEncodingSerde blockEncodingSerde = new TestingBlockEncodingSerde();
    ObjectMapper mapper = new ObjectMapperProvider().get().registerModule(new SimpleModule().addDeserializer(Type.class, new TestingTypeDeserializer(typeManager)).addSerializer(Block.class, new TestingBlockJsonSerde.Serializer(blockEncodingSerde)).addDeserializer(Block.class, new TestingBlockJsonSerde.Deserializer(blockEncodingSerde)));
    SortedRangeSet set = SortedRangeSet.all(BIGINT);
    assertEquals(set, mapper.readValue(mapper.writeValueAsString(set), SortedRangeSet.class));
    set = SortedRangeSet.none(DOUBLE);
    assertEquals(set, mapper.readValue(mapper.writeValueAsString(set), SortedRangeSet.class));
    set = SortedRangeSet.of(VARCHAR, utf8Slice("abc"));
    assertEquals(set, mapper.readValue(mapper.writeValueAsString(set), SortedRangeSet.class));
    set = SortedRangeSet.of(Range.equal(BOOLEAN, true), Range.equal(BOOLEAN, false));
    assertEquals(set, mapper.readValue(mapper.writeValueAsString(set), SortedRangeSet.class));
}

Example 10 with TestingBlockEncodingSerde

use of io.trino.spi.block.TestingBlockEncodingSerde in project trino by trinodb.

the class TestTupleDomain method testJsonSerialization.

@Test
public void testJsonSerialization() throws Exception {
    TestingTypeManager typeManager = new TestingTypeManager();
    TestingBlockEncodingSerde blockEncodingSerde = new TestingBlockEncodingSerde();
    ObjectMapper mapper = new ObjectMapperProvider().get().registerModule(new SimpleModule().addDeserializer(ColumnHandle.class, new JsonDeserializer<>() {

        @Override
        public ColumnHandle deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
            return new ObjectMapperProvider().get().readValue(jsonParser, TestingColumnHandle.class);
        }
    }).addDeserializer(Type.class, new TestingTypeDeserializer(typeManager)).addSerializer(Block.class, new TestingBlockJsonSerde.Serializer(blockEncodingSerde)).addDeserializer(Block.class, new TestingBlockJsonSerde.Deserializer(blockEncodingSerde)));
    TupleDomain<ColumnHandle> tupleDomain = TupleDomain.all();
    assertEquals(tupleDomain, mapper.readValue(mapper.writeValueAsString(tupleDomain), new TypeReference<TupleDomain<ColumnHandle>>() {
    }));
    tupleDomain = TupleDomain.none();
    assertEquals(tupleDomain, mapper.readValue(mapper.writeValueAsString(tupleDomain), new TypeReference<TupleDomain<ColumnHandle>>() {
    }));
    tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(A, NullableValue.of(BIGINT, 1L), B, NullableValue.asNull(VARCHAR)));
    assertEquals(tupleDomain, mapper.readValue(mapper.writeValueAsString(tupleDomain), new TypeReference<TupleDomain<ColumnHandle>>() {
    }));
}