Examples with ScanFixture org.apache.drill.exec.physical.impl.scan.ScanTestUtils.ScanFixture used on opensource projects

Example 21 with ScanFixture

use of org.apache.drill.exec.physical.impl.scan.ScanTestUtils.ScanFixture in project drill by apache.

the class TestScanOperExecEarlySchema method testMultipleReaders.

/**
 * Test normal case with multiple readers. These return
 * the same schema, so no schema change.
 */
@Test
public void testMultipleReaders() {
    MockNullEarlySchemaReader nullReader = new MockNullEarlySchemaReader();
    MockEarlySchemaReader reader1 = new MockEarlySchemaReader();
    reader1.batchLimit = 2;
    MockEarlySchemaReader reader2 = new MockEarlySchemaReader();
    reader2.batchLimit = 2;
    reader2.startIndex = 100;
    ScanFixture scanFixture = simpleFixture(nullReader, reader1, reader2);
    ScanOperatorExec scan = scanFixture.scanOp;
    // First batch, schema only.
    assertTrue(scan.buildSchema());
    assertEquals(1, scan.batchAccessor().schemaVersion());
    scan.batchAccessor().release();
    // Second batch.
    assertTrue(scan.next());
    assertEquals(1, reader1.batchCount);
    assertEquals(1, scan.batchAccessor().schemaVersion());
    verifyBatch(0, scan.batchAccessor().container());
    // Third batch.
    assertTrue(scan.next());
    assertEquals(2, reader1.batchCount);
    assertEquals(1, scan.batchAccessor().schemaVersion());
    verifyBatch(20, scan.batchAccessor().container());
    // Second reader. First batch includes data, no special first-batch
    // handling for the second reader.
    assertFalse(reader1.closeCalled);
    assertFalse(reader2.openCalled);
    assertTrue(scan.next());
    assertTrue(reader1.closeCalled);
    assertTrue(reader2.openCalled);
    assertEquals(1, reader2.batchCount);
    assertEquals(1, scan.batchAccessor().schemaVersion());
    verifyBatch(100, scan.batchAccessor().container());
    // Second batch from second reader.
    assertTrue(scan.next());
    assertEquals(2, reader2.batchCount);
    assertEquals(1, scan.batchAccessor().schemaVersion());
    verifyBatch(120, scan.batchAccessor().container());
    // EOF
    assertFalse(scan.next());
    assertTrue(reader2.closeCalled);
    assertEquals(0, scan.batchAccessor().rowCount());
    scanFixture.close();
}

Example 22 with ScanFixture

use of org.apache.drill.exec.physical.impl.scan.ScanTestUtils.ScanFixture in project drill by apache.

the class TestScanOperExecLimit method testLimitOnEOF.

/**
 * LIMIT 100, at EOF of the first reader.
 */
@Test
public void testLimitOnEOF() {
    Mock50RowReader reader1 = new Mock50RowReader();
    Mock50RowReader reader2 = new Mock50RowReader();
    BaseScanFixtureBuilder builder = simpleBuilder(reader1, reader2);
    builder.builder.limit(100);
    ScanFixture scanFixture = builder.build();
    ScanOperatorExec scan = scanFixture.scanOp;
    assertTrue(scan.buildSchema());
    assertTrue(scan.next());
    BatchAccessor batch = scan.batchAccessor();
    assertEquals(50, batch.rowCount());
    batch.release();
    assertTrue(scan.next());
    batch = scan.batchAccessor();
    assertEquals(50, batch.rowCount());
    batch.release();
    // No second reader
    assertFalse(scan.next());
    scanFixture.close();
    assertTrue(reader1.openCalled);
    assertFalse(reader2.openCalled);
}

Example 23 with ScanFixture

use of org.apache.drill.exec.physical.impl.scan.ScanTestUtils.ScanFixture in project drill by apache.

the class TestScanOperExecLimit method testLimitOnScondBatch.

/**
 * LIMIT 75, halfway through second batch.
 */
@Test
public void testLimitOnScondBatch() {
    Mock50RowReader reader1 = new Mock50RowReader();
    Mock50RowReader reader2 = new Mock50RowReader();
    BaseScanFixtureBuilder builder = simpleBuilder(reader1, reader2);
    builder.builder.limit(75);
    ScanFixture scanFixture = builder.build();
    ScanOperatorExec scan = scanFixture.scanOp;
    assertTrue(scan.buildSchema());
    assertTrue(scan.next());
    BatchAccessor batch = scan.batchAccessor();
    assertEquals(50, batch.rowCount());
    batch.release();
    assertTrue(scan.next());
    batch = scan.batchAccessor();
    assertEquals(25, batch.rowCount());
    batch.release();
    // No second reader
    assertFalse(scan.next());
    scanFixture.close();
    assertTrue(reader1.openCalled);
    assertFalse(reader2.openCalled);
}

Example 24 with ScanFixture

use of org.apache.drill.exec.physical.impl.scan.ScanTestUtils.ScanFixture in project drill by apache.

the class TestScanOperExecOverflow method runOverflowTest.

private void runOverflowTest(boolean eofWithData) {
    OverflowReader reader1 = new OverflowReader();
    reader1.batchLimit = 2;
    reader1.reportEofWithOverflow = eofWithData;
    MockEarlySchemaReader reader2 = new MockEarlySchemaReader();
    reader2.batchLimit = 2;
    BaseScanFixtureBuilder builder = new BaseScanFixtureBuilder();
    builder.projectAll();
    builder.addReader(reader1);
    builder.addReader(reader2);
    // Want overflow, set size and row counts at their limits.
    builder.builder.batchByteLimit(ScanSchemaOrchestrator.MAX_BATCH_BYTE_SIZE);
    builder.builder.batchRecordLimit(ScanSchemaOrchestrator.MAX_BATCH_ROW_COUNT);
    ScanFixture scanFixture = builder.build();
    ScanOperatorExec scan = scanFixture.scanOp;
    assertTrue(scan.buildSchema());
    assertEquals(1, scan.batchAccessor().schemaVersion());
    scan.batchAccessor().release();
    // Second batch. Should be 1 less than the reader's row
    // count because the loader has its own one-row lookahead batch.
    assertTrue(scan.next());
    assertEquals(1, reader1.batchCount);
    assertEquals(1, scan.batchAccessor().schemaVersion());
    int prevRowCount = scan.batchAccessor().rowCount();
    assertEquals(reader1.rowCount - 1, prevRowCount);
    scan.batchAccessor().release();
    // Third batch, adds more data to the lookahead batch. Also overflows
    // so returned records is one less than total produced so far minus
    // those returned earlier.
    assertTrue(scan.next());
    assertEquals(2, reader1.batchCount);
    assertEquals(1, scan.batchAccessor().schemaVersion());
    assertEquals(reader1.rowCount - prevRowCount - 1, scan.batchAccessor().rowCount());
    scan.batchAccessor().release();
    int prevReaderRowCount = reader1.rowCount;
    // Third batch. Returns the overflow row from the second batch of
    // the first reader.
    assertTrue(scan.next());
    assertEquals(eofWithData ? 2 : 3, reader1.batchCount);
    assertEquals(1, scan.batchAccessor().schemaVersion());
    assertEquals(1, scan.batchAccessor().rowCount());
    assertEquals(prevReaderRowCount, reader1.rowCount);
    scan.batchAccessor().release();
    // Second reader.
    assertTrue(scan.next());
    assertEquals(2, scan.batchAccessor().schemaVersion());
    scan.batchAccessor().release();
    // Second batch from second reader.
    assertTrue(scan.next());
    assertEquals(2, reader2.batchCount);
    assertEquals(2, scan.batchAccessor().schemaVersion());
    scan.batchAccessor().release();
    // EOF
    assertFalse(scan.next());
    assertEquals(0, scan.batchAccessor().rowCount());
    scanFixture.close();
}

Example 25 with ScanFixture

use of org.apache.drill.exec.physical.impl.scan.ScanTestUtils.ScanFixture in project drill by apache.

the class TestMockRowReader method testOverflow.

/**
 * Test a mock varchar column large enough to cause vector overflow.
 */
@Test
public void testOverflow() {
    int rowCount = ValueVector.MAX_ROW_COUNT;
    MockTableDef.MockColumn[] cols = new MockTableDef.MockColumn[] { new MockTableDef.MockColumn("a", MinorType.INT, DataMode.REQUIRED, null, null, null, null, null, null), new MockTableDef.MockColumn("b", MinorType.VARCHAR, DataMode.REQUIRED, 1000, null, null, null, null, null) };
    MockTableDef.MockScanEntry entry = new MockTableDef.MockScanEntry(rowCount, true, null, null, cols);
    MockSubScanPOP config = new MockSubScanPOP("dummy", true, Collections.singletonList(entry));
    ManagedReader<SchemaNegotiator> reader = new ExtendedMockBatchReader(entry);
    @SuppressWarnings("unchecked") List<ManagedReader<SchemaNegotiator>> readers = Collections.singletonList(reader);
    // Create options and the scan operator
    ScanFixture mockBatch = buildScan(config, readers);
    ScanOperatorExec scan = mockBatch.scanOp;
    // First batch: build schema. The reader helps: it returns an
    // empty first batch.
    assertTrue(scan.buildSchema());
    assertEquals(0, scan.batchAccessor().rowCount());
    // Next call, return with data, limited by batch size.
    int totalRowCount = 0;
    int batchCount = 0;
    while (scan.next()) {
        assertTrue(scan.batchAccessor().rowCount() < ValueVector.MAX_ROW_COUNT);
        BatchAccessor batchAccessor = scan.batchAccessor();
        totalRowCount += batchAccessor.rowCount();
        batchCount++;
        batchAccessor.release();
    }
    assertEquals(ValueVector.MAX_ROW_COUNT, totalRowCount);
    assertTrue(batchCount > 1);
    mockBatch.close();
}