Examples with ScanOperatorExec org.apache.drill.exec.physical.impl.scan.ScanOperatorExec used on opensource projects

Example 31 with ScanOperatorExec

use of org.apache.drill.exec.physical.impl.scan.ScanOperatorExec in project drill by apache.

the class TestScanEarlySchema method testEarlySchemaLifecycleNoSchemaBatch.

@Test
public void testEarlySchemaLifecycleNoSchemaBatch() {
    // Create a mock reader, return one batch with data.
    ReaderCreator creator = negotiator -> {
        MockEarlySchemaReader reader = new MockEarlySchemaReader(negotiator);
        reader.batchLimit = 1;
        return reader;
    };
    // Create the scan operator
    BaseScanFixtureBuilder builder = simpleBuilder(creator);
    builder.enableSchemaBatch = false;
    ScanFixture scanFixture = builder.build();
    ScanOperatorExec scan = scanFixture.scanOp;
    SingleRowSet expected = makeExpected();
    RowSetComparison verifier = new RowSetComparison(expected);
    // First batch: return with data.
    assertTrue(scan.next());
    verifier.verifyAndClearAll(fixture.wrap(scan.batchAccessor().container()));
    // EOF
    assertFalse(scan.next());
    assertEquals(0, scan.batchAccessor().rowCount());
    // Next again: no-op
    assertFalse(scan.next());
    scanFixture.close();
    // Close again: no-op
    scan.close();
}

Example 32 with ScanOperatorExec

use of org.apache.drill.exec.physical.impl.scan.ScanOperatorExec in project drill by apache.

the class TestScanEarlySchema method testEarlySchemaLifecycle.

@Test
public void testEarlySchemaLifecycle() {
    // Create a mock reader, return two batches: one schema-only, another with data.
    ObservableCreator creator = new ObservableCreator() {

        @Override
        public ManagedReader create(SchemaNegotiator negotiator) {
            MockEarlySchemaReader reader = new MockEarlySchemaReader(negotiator);
            reader.batchLimit = 1;
            return reader;
        }
    };
    ScanFixture scanFixture = simpleFixture(creator);
    ScanOperatorExec scan = scanFixture.scanOp;
    SingleRowSet expected = makeExpected();
    RowSetComparison verifier = new RowSetComparison(expected);
    // First batch: return schema.
    assertTrue(scan.buildSchema());
    MockEarlySchemaReader reader = creator.reader();
    assertEquals(0, reader.batchCount);
    assertEquals(expected.batchSchema(), scan.batchAccessor().schema());
    assertEquals(0, scan.batchAccessor().rowCount());
    // Next call, return with data.
    assertTrue(scan.next());
    verifier.verifyAndClearAll(fixture.wrap(scan.batchAccessor().container()));
    // EOF
    assertFalse(scan.next());
    assertEquals(0, scan.batchAccessor().rowCount());
    // Next again: no-op
    assertFalse(scan.next());
    scanFixture.close();
    // Close again: no-op
    scan.close();
}

Example 33 with ScanOperatorExec

use of org.apache.drill.exec.physical.impl.scan.ScanOperatorExec in project drill by apache.

the class TestScanLimit method testLimitOnBatchEnd.

/**
 * LIMIT 50, same as batch size, to check boundary conditions.
 */
@Test
public void testLimitOnBatchEnd() {
    TestFixture fixture = new TestFixture(50);
    ScanOperatorExec scan = fixture.scan;
    assertTrue(scan.buildSchema());
    assertTrue(scan.next());
    BatchAccessor batch = scan.batchAccessor();
    assertEquals(50, batch.rowCount());
    batch.release();
    // No second batch or second reader
    assertFalse(scan.next());
    fixture.close();
    // Only the first of the two readers were created.
    assertEquals(1, fixture.createCount());
}

Example 34 with ScanOperatorExec

use of org.apache.drill.exec.physical.impl.scan.ScanOperatorExec in project drill by apache.

the class TestScanOverflow method runOverflowTest.

private void runOverflowTest(boolean eofWithData) {
    ObservableCreator creator1 = new ObservableCreator() {

        @Override
        public ManagedReader create(SchemaNegotiator negotiator) {
            OverflowReader reader = new OverflowReader(negotiator);
            reader.batchLimit = 2;
            reader.reportEofWithOverflow = eofWithData;
            return reader;
        }
    };
    ObservableCreator creator2 = new ObservableCreator() {

        @Override
        public ManagedReader create(SchemaNegotiator negotiator) {
            OverflowReader reader = new OverflowReader(negotiator);
            reader.batchLimit = 2;
            return reader;
        }
    };
    BaseScanFixtureBuilder builder = new BaseScanFixtureBuilder(fixture);
    builder.projectAll();
    builder.addReader(creator1);
    builder.addReader(creator2);
    // Want overflow, set size and row counts at their limits.
    builder.builder.batchByteLimit(ScanLifecycleBuilder.MAX_BATCH_BYTE_SIZE);
    builder.builder.batchRecordLimit(ScanLifecycleBuilder.MAX_BATCH_ROW_COUNT);
    ScanFixture scanFixture = builder.build();
    ScanOperatorExec scan = scanFixture.scanOp;
    assertTrue(scan.buildSchema());
    assertEquals(1, scan.batchAccessor().schemaVersion());
    scan.batchAccessor().release();
    // Second batch. Should be 1 less than the reader's row
    // count because the loader has its own one-row lookahead batch.
    assertTrue(scan.next());
    OverflowReader reader1 = creator1.reader();
    assertEquals(1, reader1.batchCount);
    assertEquals(1, scan.batchAccessor().schemaVersion());
    int prevRowCount = scan.batchAccessor().rowCount();
    assertEquals(reader1.rowCount - 1, prevRowCount);
    scan.batchAccessor().release();
    // Third batch, adds more data to the lookahead batch. Also overflows
    // so returned records is one less than total produced so far minus
    // those returned earlier.
    assertTrue(scan.next());
    assertEquals(2, reader1.batchCount);
    assertEquals(1, scan.batchAccessor().schemaVersion());
    assertEquals(reader1.rowCount - prevRowCount - 1, scan.batchAccessor().rowCount());
    scan.batchAccessor().release();
    int prevReaderRowCount = reader1.rowCount;
    // Third batch. Returns the overflow row from the second batch of
    // the first reader.
    assertTrue(scan.next());
    assertEquals(eofWithData ? 2 : 3, reader1.batchCount);
    assertEquals(1, scan.batchAccessor().schemaVersion());
    assertEquals(1, scan.batchAccessor().rowCount());
    assertEquals(prevReaderRowCount, reader1.rowCount);
    scan.batchAccessor().release();
    // Second reader.
    assertTrue(scan.next());
    assertEquals(1, scan.batchAccessor().schemaVersion());
    scan.batchAccessor().release();
    // Second batch from second reader.
    assertTrue(scan.next());
    OverflowReader reader2 = creator2.reader();
    assertEquals(2, reader2.batchCount);
    assertEquals(1, scan.batchAccessor().schemaVersion());
    scan.batchAccessor().release();
    // Third batch. Returns the overflow row from the second batch of
    // the second reader.
    assertTrue(scan.next());
    assertEquals(3, reader2.batchCount);
    assertEquals(1, scan.batchAccessor().schemaVersion());
    assertEquals(1, scan.batchAccessor().rowCount());
    assertEquals(prevReaderRowCount, reader2.rowCount);
    scan.batchAccessor().release();
    // EOF
    assertFalse(scan.next());
    assertEquals(0, scan.batchAccessor().rowCount());
    scanFixture.close();
}

Example 35 with ScanOperatorExec

use of org.apache.drill.exec.physical.impl.scan.ScanOperatorExec in project drill by apache.

the class TestFileScan method testLateSchemaFileWildcards.

@Test
public void testLateSchemaFileWildcards() {
    // Create a mock reader, return two batches: one schema-only, another with data.
    ReaderCreator creator = negotiator -> {
        MockLateSchemaReader reader = new MockLateSchemaReader(negotiator);
        reader.batchLimit = 1;
        reader.returnDataOnFirst = true;
        return reader;
    };
    // Create the scan operator
    FileScanFixtureBuilder builder = new FileScanFixtureBuilder();
    builder.projectAllWithImplicit(3);
    builder.addReader(creator);
    ScanFixture scanFixture = builder.build();
    ScanOperatorExec scan = scanFixture.scanOp;
    // First batch: build schema. The reader helps: it returns an
    // empty first batch.
    assertTrue(scan.buildSchema());
    assertEquals(0, scan.batchAccessor().rowCount());
    // Create the expected result.
    TupleMetadata expectedSchema = new SchemaBuilder().add("a", MinorType.INT).addNullable("b", MinorType.VARCHAR).add(FileScanUtils.FULLY_QUALIFIED_NAME_COL, MinorType.VARCHAR).add(FileScanUtils.FILE_PATH_COL, MinorType.VARCHAR).add(FileScanUtils.FILE_NAME_COL, MinorType.VARCHAR).add(FileScanUtils.SUFFIX_COL, MinorType.VARCHAR).addNullable(FileScanUtils.partitionColName(0), MinorType.VARCHAR).addNullable(FileScanUtils.partitionColName(1), MinorType.VARCHAR).addNullable(FileScanUtils.partitionColName(2), MinorType.VARCHAR).buildSchema();
    SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(10, "fred", MOCK_FILE_FQN, MOCK_FILE_DIR_PATH, MOCK_FILE_NAME, MOCK_SUFFIX, MOCK_DIR0, MOCK_DIR1, null).addRow(20, "wilma", MOCK_FILE_FQN, MOCK_FILE_DIR_PATH, MOCK_FILE_NAME, MOCK_SUFFIX, MOCK_DIR0, MOCK_DIR1, null).build();
    assertEquals(expected.batchSchema(), scan.batchAccessor().schema());
    // Next call, return with data.
    assertTrue(scan.next());
    RowSetUtilities.verify(expected, fixture.wrap(scan.batchAccessor().container()));
    // EOF
    assertFalse(scan.next());
    assertEquals(0, scan.batchAccessor().rowCount());
    scanFixture.close();
}