Examples with DirectRowSet org.apache.drill.exec.physical.rowSet.DirectRowSet used on opensource projects

Example 21 with DirectRowSet

use of org.apache.drill.exec.physical.rowSet.DirectRowSet in project drill by apache.

the class QueryRowSetIterator method printAll.

public void printAll() {
    for (DirectRowSet rowSet : this) {
        RowSetFormatter.print(rowSet);
        rowSet.clear();
    }
}

Example 22 with DirectRowSet

use of org.apache.drill.exec.physical.rowSet.DirectRowSet in project drill by apache.

the class TestCsvWithHeaders method testWildcardAndPartitionsMultiFiles.

/**
 * Test the use of partition columns with the wildcard. This works for file
 * metadata columns, but confuses the project operator when used for
 * partition columns. DRILL-7080. Still broken in V3 because this appears
 * to be a Project operator issue, not reader issue. Not that the
 * partition column moves after data columns.
 */
@Test
public void testWildcardAndPartitionsMultiFiles() {
    String sql = "SELECT *, dir0, dir1 FROM `dfs.data`.`%s`";
    Iterator<DirectRowSet> iter = client.queryBuilder().sql(sql, PART_DIR).rowSetIterator();
    TupleMetadata expectedSchema = new SchemaBuilder().add("a", MinorType.VARCHAR).add("b", MinorType.VARCHAR).add("c", MinorType.VARCHAR).addNullable("dir0", MinorType.VARCHAR).addNullable("dir1", MinorType.VARCHAR).addNullable("dir00", MinorType.VARCHAR).addNullable("dir10", MinorType.VARCHAR).buildSchema();
    RowSet rowSet;
    if (SCHEMA_BATCH_ENABLED) {
        // First batch is empty; just carries the schema.
        assertTrue(iter.hasNext());
        rowSet = iter.next();
        RowSetUtilities.verify(new RowSetBuilder(client.allocator(), expectedSchema).build(), rowSet);
    }
    // Read the two batches.
    for (int i = 0; i < 2; i++) {
        assertTrue(iter.hasNext());
        rowSet = iter.next();
        // Figure out which record this is and test accordingly.
        RowSetReader reader = rowSet.reader();
        assertTrue(reader.next());
        String aCol = reader.scalar("a").getString();
        if (aCol.equals("10")) {
            RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema).addRow("10", "foo", "bar", null, null, null, null).build();
            RowSetUtilities.verify(expected, rowSet);
        } else {
            RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema).addRow("20", "fred", "wilma", NESTED_DIR, null, NESTED_DIR, null).build();
            RowSetUtilities.verify(expected, rowSet);
        }
    }
    assertFalse(iter.hasNext());
}

Example 23 with DirectRowSet

use of org.apache.drill.exec.physical.rowSet.DirectRowSet in project drill by apache.

the class TestCsvWithHeaders method doTestExplicitPartitionsMultiFiles.

/**
 * Test using partition columns with partitioned files in V3. Although the
 * file is nested to one level, both dir0 and dir1 are nullable VARCHAR.
 * See {@link TestPartitionRace} to show that the types and schemas
 * are consistent even when used across multiple scans.
 */
@Test
public void doTestExplicitPartitionsMultiFiles() {
    String sql = "SELECT a, b, c, dir0, dir1 FROM `dfs.data`.`%s`";
    Iterator<DirectRowSet> iter = client.queryBuilder().sql(sql, PART_DIR).rowSetIterator();
    TupleMetadata expectedSchema = new SchemaBuilder().add("a", MinorType.VARCHAR).add("b", MinorType.VARCHAR).add("c", MinorType.VARCHAR).addNullable("dir0", MinorType.VARCHAR).addNullable("dir1", MinorType.VARCHAR).buildSchema();
    RowSet rowSet;
    if (SCHEMA_BATCH_ENABLED) {
        // First batch is empty; just carries the schema.
        assertTrue(iter.hasNext());
        rowSet = iter.next();
        RowSetUtilities.verify(new RowSetBuilder(client.allocator(), expectedSchema).build(), rowSet);
    }
    // Read the two batches.
    for (int i = 0; i < 2; i++) {
        assertTrue(iter.hasNext());
        rowSet = iter.next();
        // Figure out which record this is and test accordingly.
        RowSetReader reader = rowSet.reader();
        assertTrue(reader.next());
        String aCol = reader.scalar("a").getString();
        if (aCol.equals("10")) {
            RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema).addRow("10", "foo", "bar", null, null).build();
            RowSetUtilities.verify(expected, rowSet);
        } else {
            RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema).addRow("20", "fred", "wilma", NESTED_DIR, null).build();
            RowSetUtilities.verify(expected, rowSet);
        }
    }
    assertFalse(iter.hasNext());
}

Example 24 with DirectRowSet

use of org.apache.drill.exec.physical.rowSet.DirectRowSet in project drill by apache.

the class TestPartitionRace method testNoRace.

/**
 * V3 computes partition depth in the group scan (which sees all files), and
 * so the partition column count does not vary across scans. Also, V3 puts
 * partition columns at the end of the row so that data columns don't
 * "jump around" when files are shifted to a new partition depth.
 */
@Test
public void testNoRace() throws IOException {
    String sql = "SELECT * FROM `dfs.data`.`%s`";
    TupleMetadata expectedSchema = new SchemaBuilder().add("a", MinorType.VARCHAR).add("b", MinorType.VARCHAR).add("c", MinorType.VARCHAR).addNullable("dir0", MinorType.VARCHAR).buildSchema();
    try {
        enableMultiScan();
        // Loop to run the query 10 times or until we see both files
        // in the first position.
        boolean sawRootFirst = false;
        boolean sawNestedFirst = false;
        for (int i = 0; i < 10; i++) {
            Iterator<DirectRowSet> iter = client.queryBuilder().sql(sql, PART_DIR).rowSetIterator();
            RowSet rowSet;
            if (SCHEMA_BATCH_ENABLED) {
                // First batch is empty; just carries the schema.
                assertTrue(iter.hasNext());
                rowSet = iter.next();
                assertEquals(0, rowSet.rowCount());
                rowSet.clear();
            }
            for (int j = 0; j < 2; j++) {
                assertTrue(iter.hasNext());
                rowSet = iter.next();
                // Figure out which record this is and test accordingly.
                RowSetReader reader = rowSet.reader();
                assertTrue(reader.next());
                String col1 = reader.scalar("a").getString();
                if (col1.equals("10")) {
                    if (i == 0) {
                        sawRootFirst = true;
                    }
                    RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema).addRow("10", "foo", "bar", null).build();
                    RowSetUtilities.verify(expected, rowSet);
                } else {
                    if (i == 0) {
                        sawNestedFirst = true;
                    }
                    RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema).addRow("20", "fred", "wilma", NESTED_DIR).build();
                    RowSetUtilities.verify(expected, rowSet);
                }
            }
            assertFalse(iter.hasNext());
            if (sawRootFirst && sawNestedFirst) {
                // The following should appear most of the time.
                System.out.println("Both variations occurred");
                return;
            }
        }
        // If you see this, maybe something got fixed. Or, maybe the
        // min parallelization hack above stopped working.
        // Or, you were just unlucky and can try the test again.
        // We print messages, rather than using assertTrue, to avoid
        // introducing a flaky test.
        System.out.println("Some variations did not occur");
        System.out.println(String.format("Outer first: %s", sawRootFirst));
        System.out.println(String.format("Nested first: %s", sawNestedFirst));
    } finally {
        resetMultiScan();
    }
}

Example 25 with DirectRowSet

use of org.apache.drill.exec.physical.rowSet.DirectRowSet in project drill by apache.

the class TestAnalyze method verifyAnalyzeOutput.

// Helper function to verify output of ANALYZE statement
private void verifyAnalyzeOutput(String query, String message) throws Exception {
    DirectRowSet rowSet = queryBuilder().sql(query).rowSet();
    try {
        assertEquals(1, rowSet.rowCount());
        RowSetReader reader = rowSet.reader();
        assertEquals(2, reader.columnCount());
        while (reader.next()) {
            ObjectReader column = reader.column(1);
            assertEquals(message, column.isNull() ? null : column.getObject().toString());
        }
    } finally {
        rowSet.clear();
    }
}