Search in sources :

Example 6 with Predicate

use of com.facebook.presto.common.relation.Predicate in project presto by prestodb.

the class TestOrcReaderPositions method testStripeSkippingWithAppendNumber.

@Test
public void testStripeSkippingWithAppendNumber() throws Exception {
    try (TempFile tempFile = new TempFile()) {
        createMultiStripeFile(tempFile.getFile());
        // EVery stripe has 20 rows and there are total of 5 stripes
        // test reading second and fourth stripes
        OrcPredicate predicate = (numberOfRows, statisticsByColumnIndex) -> {
            if (numberOfRows == 100) {
                return true;
            }
            IntegerStatistics stats = statisticsByColumnIndex.get(0).getIntegerStatistics();
            return ((stats.getMin() == 60) && (stats.getMax() == 117)) || ((stats.getMin() == 180) && (stats.getMax() == 237));
        };
        List<Long> expectedValues = new ArrayList<>();
        expectedValues.addAll(LongStream.range(20, 40).collect(ArrayList::new, List::add, List::addAll));
        expectedValues.addAll(LongStream.range(60, 80).collect(ArrayList::new, List::add, List::addAll));
        List<Long> actualValues = new ArrayList<>();
        OrcSelectiveRecordReader reader = createCustomOrcSelectiveRecordReader(tempFile, ORC, predicate, BIGINT, MAX_BATCH_SIZE, false, true);
        assertNotNull(reader);
        Page returnPage;
        while (true) {
            returnPage = reader.getNextPage();
            if (returnPage == null) {
                break;
            }
            Block rowNumberBlock = returnPage.getBlock(1);
            for (int i = 0; i < returnPage.getPositionCount(); i++) {
                actualValues.add(rowNumberBlock.getLong(i));
            }
        }
        assertEquals(actualValues, expectedValues);
    }
}
Also used : OrcFile(org.apache.hadoop.hive.ql.io.orc.OrcFile) Page(com.facebook.presto.common.Page) OrcTester.createCustomOrcSelectiveRecordReader(com.facebook.presto.orc.OrcTester.createCustomOrcSelectiveRecordReader) Writable(org.apache.hadoop.io.Writable) Test(org.testng.annotations.Test) ORC_12(com.facebook.presto.orc.OrcTester.Format.ORC_12) OrcSerde(org.apache.hadoop.hive.ql.io.orc.OrcSerde) ByteBuffer(java.nio.ByteBuffer) Writer(org.apache.hadoop.hive.ql.io.orc.Writer) FileSinkOperator(org.apache.hadoop.hive.ql.exec.FileSinkOperator) Configuration(org.apache.hadoop.conf.Configuration) Map(java.util.Map) OrcTester.createOrcRecordWriter(com.facebook.presto.orc.OrcTester.createOrcRecordWriter) Path(org.apache.hadoop.fs.Path) Predicate(com.facebook.presto.common.relation.Predicate) RuntimeStats(com.facebook.presto.common.RuntimeStats) NullMemoryManager(org.apache.orc.NullMemoryManager) ImmutableMap(com.google.common.collect.ImmutableMap) Footer(com.facebook.presto.orc.metadata.Footer) SqlFunctionProperties(com.facebook.presto.common.function.SqlFunctionProperties) INITIAL_BATCH_SIZE(com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE) Math.min(java.lang.Math.min) Assert.assertNotNull(org.testng.Assert.assertNotNull) NOOP_ORC_AGGREGATED_MEMORY_CONTEXT(com.facebook.presto.orc.NoopOrcAggregatedMemoryContext.NOOP_ORC_AGGREGATED_MEMORY_CONTEXT) ConnectorSession(com.facebook.presto.spi.ConnectorSession) ORC(com.facebook.presto.orc.OrcEncoding.ORC) DataSize(io.airlift.units.DataSize) List(java.util.List) OrcTester.createCustomOrcRecordReader(com.facebook.presto.orc.OrcTester.createCustomOrcRecordReader) CompressionKind(com.facebook.presto.orc.metadata.CompressionKind) OrcOutputFormat(org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat) StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) MAX_BLOCK_SIZE(com.facebook.presto.orc.OrcTester.MAX_BLOCK_SIZE) Slice(io.airlift.slice.Slice) VARCHAR(com.facebook.presto.common.type.VarcharType.VARCHAR) MEGABYTE(io.airlift.units.DataSize.Unit.MEGABYTE) Assert.assertEquals(org.testng.Assert.assertEquals) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) FilterFunction(com.facebook.presto.common.predicate.FilterFunction) MAX_BATCH_SIZE(com.facebook.presto.orc.OrcReader.MAX_BATCH_SIZE) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) LongStream(java.util.stream.LongStream) BATCH_SIZE_GROWTH_FACTOR(com.facebook.presto.orc.OrcReader.BATCH_SIZE_GROWTH_FACTOR) UTF_8(java.nio.charset.StandardCharsets.UTF_8) SNAPPY(org.apache.hadoop.hive.ql.io.orc.CompressionKind.SNAPPY) NO_ENCRYPTION(com.facebook.presto.orc.DwrfEncryptionProvider.NO_ENCRYPTION) IOException(java.io.IOException) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) Field(java.lang.reflect.Field) Maps(com.google.common.collect.Maps) File(java.io.File) SettableStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector) StorageOrcFileTailSource(com.facebook.presto.orc.cache.StorageOrcFileTailSource) Serializer(org.apache.hadoop.hive.serde2.Serializer) IntegerStatistics(com.facebook.presto.orc.metadata.statistics.IntegerStatistics) Assert.assertTrue(org.testng.Assert.assertTrue) Block(com.facebook.presto.common.block.Block) SerDeException(org.apache.hadoop.hive.serde2.SerDeException) OrcTester.createSettableStructObjectInspector(com.facebook.presto.orc.OrcTester.createSettableStructObjectInspector) OrcTester.createCustomOrcSelectiveRecordReader(com.facebook.presto.orc.OrcTester.createCustomOrcSelectiveRecordReader) ArrayList(java.util.ArrayList) Block(com.facebook.presto.common.block.Block) Page(com.facebook.presto.common.Page) IntegerStatistics(com.facebook.presto.orc.metadata.statistics.IntegerStatistics) Test(org.testng.annotations.Test)

Example 7 with Predicate

use of com.facebook.presto.common.relation.Predicate in project presto by prestodb.

the class RowExpressionPredicateCompiler method definePredicateClass.

private ClassDefinition definePredicateClass(SqlFunctionProperties sqlFunctionProperties, Map<SqlFunctionId, SqlInvokedFunction> sessionFunctions, RowExpression predicate, int[] inputChannels, CallSiteBinder callSiteBinder) {
    ClassDefinition classDefinition = new ClassDefinition(a(PUBLIC, FINAL), makeClassName(Predicate.class.getSimpleName(), Optional.empty()), type(Object.class), type(Predicate.class));
    CachedInstanceBinder cachedInstanceBinder = new CachedInstanceBinder(classDefinition, callSiteBinder);
    generatePredicateMethod(sqlFunctionProperties, sessionFunctions, classDefinition, callSiteBinder, cachedInstanceBinder, predicate);
    // getInputChannels
    classDefinition.declareMethod(a(PUBLIC), "getInputChannels", type(int[].class)).getBody().append(invoke(callSiteBinder.bind(inputChannels, int[].class), "getInputChannels")).retObject();
    // constructor
    generateConstructor(classDefinition, cachedInstanceBinder);
    return classDefinition;
}
Also used : ClassDefinition(com.facebook.presto.bytecode.ClassDefinition) Predicate(com.facebook.presto.common.relation.Predicate)

Aggregations

Predicate (com.facebook.presto.common.relation.Predicate)7 Page (com.facebook.presto.common.Page)5 Block (com.facebook.presto.common.block.Block)4 SqlFunctionProperties (com.facebook.presto.common.function.SqlFunctionProperties)4 FilterFunction (com.facebook.presto.common.predicate.FilterFunction)4 RuntimeStats (com.facebook.presto.common.RuntimeStats)3 BIGINT (com.facebook.presto.common.type.BigintType.BIGINT)3 VARCHAR (com.facebook.presto.common.type.VarcharType.VARCHAR)3 NO_ENCRYPTION (com.facebook.presto.orc.DwrfEncryptionProvider.NO_ENCRYPTION)3 NOOP_ORC_AGGREGATED_MEMORY_CONTEXT (com.facebook.presto.orc.NoopOrcAggregatedMemoryContext.NOOP_ORC_AGGREGATED_MEMORY_CONTEXT)3 ORC (com.facebook.presto.orc.OrcEncoding.ORC)3 BATCH_SIZE_GROWTH_FACTOR (com.facebook.presto.orc.OrcReader.BATCH_SIZE_GROWTH_FACTOR)3 INITIAL_BATCH_SIZE (com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE)3 MAX_BATCH_SIZE (com.facebook.presto.orc.OrcReader.MAX_BATCH_SIZE)3 ORC_12 (com.facebook.presto.orc.OrcTester.Format.ORC_12)3 MAX_BLOCK_SIZE (com.facebook.presto.orc.OrcTester.MAX_BLOCK_SIZE)3 OrcTester.createCustomOrcRecordReader (com.facebook.presto.orc.OrcTester.createCustomOrcRecordReader)3 OrcTester.createCustomOrcSelectiveRecordReader (com.facebook.presto.orc.OrcTester.createCustomOrcSelectiveRecordReader)3 OrcTester.createOrcRecordWriter (com.facebook.presto.orc.OrcTester.createOrcRecordWriter)3 OrcTester.createSettableStructObjectInspector (com.facebook.presto.orc.OrcTester.createSettableStructObjectInspector)3