Search in sources :

Example 21 with ScanOperatorExec

use of org.apache.drill.exec.physical.impl.scan.ScanOperatorExec in project drill by apache.

the class TestMockRowReader method testColumnRepeat.

/**
 * Test a repeated column.
 */
@Test
public void testColumnRepeat() {
    int rowCount = 10;
    MockTableDef.MockColumn[] cols = new MockTableDef.MockColumn[] { new MockTableDef.MockColumn("a", MinorType.INT, DataMode.REQUIRED, null, null, null, null, 3, null), new MockTableDef.MockColumn("b", MinorType.VARCHAR, DataMode.REQUIRED, 10, null, null, null, null, null) };
    MockTableDef.MockScanEntry entry = new MockTableDef.MockScanEntry(rowCount, true, null, null, cols);
    MockSubScanPOP config = new MockSubScanPOP("dummy", true, Collections.singletonList(entry));
    ManagedReader<SchemaNegotiator> reader = new ExtendedMockBatchReader(entry);
    List<ManagedReader<SchemaNegotiator>> readers = Collections.singletonList(reader);
    // Create options and the scan operator
    ScanFixture mockBatch = buildScan(config, readers);
    ScanOperatorExec scan = mockBatch.scanOp;
    // First batch: build schema. The reader helps: it returns an
    // empty first batch.
    assertTrue(scan.buildSchema());
    TupleMetadata expectedSchema = new SchemaBuilder().add("a1", MinorType.INT).add("a2", MinorType.INT).add("a3", MinorType.INT).add("b", MinorType.VARCHAR, 10).build();
    BatchSchema expectedBatchSchema = new BatchSchema(SelectionVectorMode.NONE, expectedSchema.toFieldList());
    assertTrue(expectedBatchSchema.isEquivalent(scan.batchAccessor().schema()));
    assertEquals(0, scan.batchAccessor().rowCount());
    // Next call, return with data.
    assertTrue(scan.next());
    assertTrue(expectedBatchSchema.isEquivalent(scan.batchAccessor().schema()));
    assertEquals(rowCount, scan.batchAccessor().rowCount());
    scan.batchAccessor().release();
    // EOF
    assertFalse(scan.next());
    mockBatch.close();
}
Also used : ScanFixture(org.apache.drill.exec.physical.impl.scan.ScanTestUtils.ScanFixture) ManagedReader(org.apache.drill.exec.physical.impl.scan.framework.ManagedReader) BatchSchema(org.apache.drill.exec.record.BatchSchema) ScanOperatorExec(org.apache.drill.exec.physical.impl.scan.ScanOperatorExec) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) SchemaBuilder(org.apache.drill.exec.record.metadata.SchemaBuilder) SchemaNegotiator(org.apache.drill.exec.physical.impl.scan.framework.SchemaNegotiator) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test) UnlikelyTest(org.apache.drill.categories.UnlikelyTest)

Example 22 with ScanOperatorExec

use of org.apache.drill.exec.physical.impl.scan.ScanOperatorExec in project drill by apache.

the class TestMockRowReader method testBatchSize.

/**
 * Verify limit on individual batch size (limiting row count per batch).
 */
@Test
public void testBatchSize() {
    int rowCount = 20;
    int batchSize = 10;
    MockTableDef.MockColumn[] cols = new MockTableDef.MockColumn[] { new MockTableDef.MockColumn("a", MinorType.INT, DataMode.REQUIRED, null, null, null, null, null, null), new MockTableDef.MockColumn("b", MinorType.VARCHAR, DataMode.REQUIRED, 10, null, null, null, null, null) };
    MockTableDef.MockScanEntry entry = new MockTableDef.MockScanEntry(rowCount, true, batchSize, null, cols);
    MockSubScanPOP config = new MockSubScanPOP("dummy", true, Collections.singletonList(entry));
    ManagedReader<SchemaNegotiator> reader = new ExtendedMockBatchReader(entry);
    List<ManagedReader<SchemaNegotiator>> readers = Collections.singletonList(reader);
    // Create options and the scan operator
    ScanFixture mockBatch = buildScan(config, readers);
    ScanOperatorExec scan = mockBatch.scanOp;
    // First batch: build schema. The reader helps: it returns an
    // empty first batch.
    assertTrue(scan.buildSchema());
    assertEquals(0, scan.batchAccessor().rowCount());
    // Next call, return with data, limited by batch size.
    assertTrue(scan.next());
    assertEquals(batchSize, scan.batchAccessor().rowCount());
    scan.batchAccessor().release();
    assertTrue(scan.next());
    assertEquals(batchSize, scan.batchAccessor().rowCount());
    scan.batchAccessor().release();
    // EOF
    assertFalse(scan.next());
    mockBatch.close();
}
Also used : ScanFixture(org.apache.drill.exec.physical.impl.scan.ScanTestUtils.ScanFixture) ManagedReader(org.apache.drill.exec.physical.impl.scan.framework.ManagedReader) ScanOperatorExec(org.apache.drill.exec.physical.impl.scan.ScanOperatorExec) SchemaNegotiator(org.apache.drill.exec.physical.impl.scan.framework.SchemaNegotiator) SubOperatorTest(org.apache.drill.test.SubOperatorTest) Test(org.junit.Test) UnlikelyTest(org.apache.drill.categories.UnlikelyTest)

Example 23 with ScanOperatorExec

use of org.apache.drill.exec.physical.impl.scan.ScanOperatorExec in project drill by apache.

the class TestScanLateSchema method testLateSchemaDataOnFirst.

@Test
public void testLateSchemaDataOnFirst() {
    // Create a mock reader, return two batches: one schema-only, another with data.
    ReaderCreator creator = negotiator -> {
        MockLateSchemaReader reader = new MockLateSchemaReader(negotiator);
        reader.batchLimit = 1;
        reader.returnDataOnFirst = true;
        return reader;
    };
    ScanFixture scanFixture = simpleFixture(creator);
    ScanOperatorExec scan = scanFixture.scanOp;
    // First batch: build schema. The reader helps: it returns an
    // empty first batch.
    assertTrue(scan.buildSchema());
    assertEquals(0, scan.batchAccessor().rowCount());
    SingleRowSet expected = makeExpected();
    RowSetComparison verifier = new RowSetComparison(expected);
    assertEquals(expected.batchSchema(), scan.batchAccessor().schema());
    // Next call, return with data.
    assertTrue(scan.next());
    verifier.verifyAndClearAll(fixture.wrap(scan.batchAccessor().container()));
    // EOF
    assertFalse(scan.next());
    assertEquals(0, scan.batchAccessor().rowCount());
    scanFixture.close();
}
Also used : ColumnBuilder(org.apache.drill.exec.record.metadata.ColumnBuilder) RowSetUtilities(org.apache.drill.test.rowSet.RowSetUtilities) Assert.assertTrue(org.junit.Assert.assertTrue) SchemaBuilder(org.apache.drill.exec.record.metadata.SchemaBuilder) Test(org.junit.Test) Category(org.junit.experimental.categories.Category) MaterializedField(org.apache.drill.exec.record.MaterializedField) RowSetLoader(org.apache.drill.exec.physical.resultSet.RowSetLoader) RowSetComparison(org.apache.drill.test.rowSet.RowSetComparison) DataMode(org.apache.drill.common.types.TypeProtos.DataMode) Assert.assertNull(org.junit.Assert.assertNull) Assert.assertFalse(org.junit.Assert.assertFalse) EvfTest(org.apache.drill.categories.EvfTest) MinorType(org.apache.drill.common.types.TypeProtos.MinorType) SingleRowSet(org.apache.drill.exec.physical.rowSet.RowSet.SingleRowSet) ScanOperatorExec(org.apache.drill.exec.physical.impl.scan.ScanOperatorExec) Assert.assertEquals(org.junit.Assert.assertEquals) SingleRowSet(org.apache.drill.exec.physical.rowSet.RowSet.SingleRowSet) RowSetComparison(org.apache.drill.test.rowSet.RowSetComparison) ScanOperatorExec(org.apache.drill.exec.physical.impl.scan.ScanOperatorExec) Test(org.junit.Test) EvfTest(org.apache.drill.categories.EvfTest)

Example 24 with ScanOperatorExec

use of org.apache.drill.exec.physical.impl.scan.ScanOperatorExec in project drill by apache.

the class TestScanOuputSchema method testStrictProvidedSchemaWithWildcardAndSpecialCols.

@Test
public void testStrictProvidedSchemaWithWildcardAndSpecialCols() {
    TupleMetadata providedSchema = new SchemaBuilder().add("a", // Projected, in reader
    MinorType.INT).add("d", // Projected, not in reader
    MinorType.BIGINT).add("e", // Not projected, not in reader
    MinorType.BIGINT).buildSchema();
    providedSchema.metadata("d").setDefaultValue("20");
    providedSchema.metadata("e").setDefaultValue("30");
    providedSchema.setProperty(TupleMetadata.IS_STRICT_SCHEMA_PROP, Boolean.TRUE.toString());
    providedSchema.metadata("a").setBooleanProperty(ColumnMetadata.EXCLUDE_FROM_WILDCARD, true);
    BaseScanFixtureBuilder builder = new BaseScanFixtureBuilder(fixture);
    // Project schema only
    builder.setProjection(RowSetTestUtils.projectAll());
    builder.addReader(negotiator -> new MockSimpleReader(negotiator));
    builder.builder.providedSchema(providedSchema);
    builder.builder.nullType(Types.optional(MinorType.VARCHAR));
    ScanFixture scanFixture = builder.build();
    ScanOperatorExec scan = scanFixture.scanOp;
    TupleMetadata expectedSchema = new SchemaBuilder().add("d", MinorType.BIGINT).add("e", MinorType.BIGINT).buildSchema();
    // Initial schema
    assertTrue(scan.buildSchema());
    {
        SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).build();
        RowSetUtilities.verify(expected, fixture.wrap(scan.batchAccessor().container()));
    }
    // Batch with defaults and null types
    assertTrue(scan.next());
    {
        SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(20L, 30L).build();
        RowSetUtilities.verify(expected, fixture.wrap(scan.batchAccessor().container()));
    }
    assertFalse(scan.next());
    scanFixture.close();
}
Also used : SingleRowSet(org.apache.drill.exec.physical.rowSet.RowSet.SingleRowSet) ScanOperatorExec(org.apache.drill.exec.physical.impl.scan.ScanOperatorExec) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) SchemaBuilder(org.apache.drill.exec.record.metadata.SchemaBuilder) Test(org.junit.Test) EvfTest(org.apache.drill.categories.EvfTest)

Example 25 with ScanOperatorExec

use of org.apache.drill.exec.physical.impl.scan.ScanOperatorExec in project drill by apache.

the class TestScanOuputSchema method testProvidedSchemaWithWildcard.

/**
 * Test non-strict specified schema, with a wildcard, with extra
 * reader columns. Reader columns are included in output.
 */
@Test
public void testProvidedSchemaWithWildcard() {
    TupleMetadata providedSchema = new SchemaBuilder().add("a", // Projected, in reader
    MinorType.INT).add("d", // Projected, not in reader
    MinorType.BIGINT).add("e", // Not projected, not in reader
    MinorType.BIGINT).buildSchema();
    providedSchema.metadata("d").setDefaultValue("20");
    providedSchema.metadata("e").setDefaultValue("30");
    BaseScanFixtureBuilder builder = new BaseScanFixtureBuilder(fixture);
    builder.setProjection(RowSetTestUtils.projectAll());
    builder.addReader(negotiator -> new MockSimpleReader(negotiator));
    builder.builder.providedSchema(providedSchema);
    builder.builder.nullType(Types.optional(MinorType.VARCHAR));
    ScanFixture scanFixture = builder.build();
    ScanOperatorExec scan = scanFixture.scanOp;
    TupleMetadata expectedSchema = new SchemaBuilder().add("a", MinorType.INT).add("d", MinorType.BIGINT).add("e", MinorType.BIGINT).add("b", MinorType.VARCHAR).add("c", MinorType.VARCHAR).buildSchema();
    // Initial schema
    assertTrue(scan.buildSchema());
    {
        SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).build();
        RowSetUtilities.verify(expected, fixture.wrap(scan.batchAccessor().container()));
    }
    // Batch with defaults and null types
    assertTrue(scan.next());
    {
        SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(10, 20L, 30L, "foo", "bar").build();
        RowSetUtilities.verify(expected, fixture.wrap(scan.batchAccessor().container()));
    }
    assertFalse(scan.next());
    scanFixture.close();
}
Also used : SingleRowSet(org.apache.drill.exec.physical.rowSet.RowSet.SingleRowSet) ScanOperatorExec(org.apache.drill.exec.physical.impl.scan.ScanOperatorExec) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) SchemaBuilder(org.apache.drill.exec.record.metadata.SchemaBuilder) Test(org.junit.Test) EvfTest(org.apache.drill.categories.EvfTest)

Aggregations

ScanOperatorExec (org.apache.drill.exec.physical.impl.scan.ScanOperatorExec)49 Test (org.junit.Test)47 EvfTest (org.apache.drill.categories.EvfTest)35 SingleRowSet (org.apache.drill.exec.physical.rowSet.RowSet.SingleRowSet)22 SchemaBuilder (org.apache.drill.exec.record.metadata.SchemaBuilder)19 RowSetUtilities (org.apache.drill.test.rowSet.RowSetUtilities)16 Assert.assertEquals (org.junit.Assert.assertEquals)16 Assert.assertFalse (org.junit.Assert.assertFalse)16 Assert.assertTrue (org.junit.Assert.assertTrue)16 Category (org.junit.experimental.categories.Category)16 TupleMetadata (org.apache.drill.exec.record.metadata.TupleMetadata)13 RowSetComparison (org.apache.drill.test.rowSet.RowSetComparison)12 DataMode (org.apache.drill.common.types.TypeProtos.DataMode)11 MinorType (org.apache.drill.common.types.TypeProtos.MinorType)11 RowSetLoader (org.apache.drill.exec.physical.resultSet.RowSetLoader)11 MaterializedField (org.apache.drill.exec.record.MaterializedField)11 UserException (org.apache.drill.common.exceptions.UserException)9 BatchAccessor (org.apache.drill.exec.physical.impl.protocol.BatchAccessor)7 ManagedReader (org.apache.drill.exec.physical.impl.scan.framework.ManagedReader)6 ColumnBuilder (org.apache.drill.exec.record.metadata.ColumnBuilder)6