Search in sources :

Example 16 with DirectRowSet

use of org.apache.drill.exec.physical.rowSet.DirectRowSet in project drill by apache.

the class TestJdbcWriterWithH2 method testBasicCTAS.

@Test
public void testBasicCTAS() throws Exception {
    String query = String.format("CREATE TABLE %s (ID, NAME) AS SELECT * FROM (VALUES(1,2), (3,4))", TEST_TABLE);
    // Create the table and insert the values
    QuerySummary insertResults = queryBuilder().sql(query).run();
    try {
        assertTrue(insertResults.succeeded());
        // Query the table to see if the insertion was successful
        String testQuery = String.format("SELECT * FROM %s", TEST_TABLE);
        DirectRowSet results = queryBuilder().sql(testQuery).rowSet();
        TupleMetadata expectedSchema = new SchemaBuilder().add("ID", MinorType.BIGINT, DataMode.OPTIONAL).add("NAME", MinorType.BIGINT, DataMode.OPTIONAL).buildSchema();
        RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema).addRow(1L, 2L).addRow(3L, 4L).build();
        RowSetUtilities.verify(expected, results);
    } finally {
        QuerySummary dropResults = queryBuilder().sql(DROP_TEST_TABLE).run();
        assertTrue(dropResults.succeeded());
    }
}
Also used : RowSetBuilder(org.apache.drill.exec.physical.rowSet.RowSetBuilder) QuerySummary(org.apache.drill.test.QueryBuilder.QuerySummary) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) DirectRowSet(org.apache.drill.exec.physical.rowSet.DirectRowSet) SchemaBuilder(org.apache.drill.exec.record.metadata.SchemaBuilder) DirectRowSet(org.apache.drill.exec.physical.rowSet.DirectRowSet) RowSet(org.apache.drill.exec.physical.rowSet.RowSet) ClusterTest(org.apache.drill.test.ClusterTest) Test(org.junit.Test)

Example 17 with DirectRowSet

use of org.apache.drill.exec.physical.rowSet.DirectRowSet in project drill by apache.

the class TestCsvWithoutHeaders method testPartitionExpansion.

/**
 * Test partition expansion.
 * <p>
 * V3, as in V2 before Drill 1.12, puts partition columns after
 * data columns (so that data columns don't shift positions if
 * files are nested to another level.)
 */
@Test
public void testPartitionExpansion() throws IOException {
    String sql = "SELECT * FROM `dfs.data`.`%s`";
    Iterator<DirectRowSet> iter = client.queryBuilder().sql(sql, PART_DIR).rowSetIterator();
    TupleMetadata expectedSchema = new SchemaBuilder().addArray("columns", MinorType.VARCHAR).addNullable("dir0", MinorType.VARCHAR).buildSchema();
    RowSet rowSet;
    if (SCHEMA_BATCH_ENABLED) {
        // First batch is empty; just carries the schema.
        assertTrue(iter.hasNext());
        rowSet = iter.next();
        assertEquals(0, rowSet.rowCount());
        rowSet.clear();
    }
    // Read the two data batches.
    for (int i = 0; i < 2; i++) {
        assertTrue(iter.hasNext());
        rowSet = iter.next();
        // Figure out which record this is and test accordingly.
        RowSetReader reader = rowSet.reader();
        assertTrue(reader.next());
        ArrayReader ar = reader.array(0);
        assertTrue(ar.next());
        String col1 = ar.scalar().getString();
        if (col1.equals("10")) {
            RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema).addRow(strArray("10", "foo", "bar"), null).addRow(strArray("20", "fred", "wilma"), null).build();
            RowSetUtilities.verify(expected, rowSet);
        } else {
            RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema).addRow(strArray("30", "barney", "betty"), NESTED_DIR).build();
            RowSetUtilities.verify(expected, rowSet);
        }
    }
    assertFalse(iter.hasNext());
}
Also used : ArrayReader(org.apache.drill.exec.vector.accessor.ArrayReader) RowSetBuilder(org.apache.drill.exec.physical.rowSet.RowSetBuilder) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) DirectRowSet(org.apache.drill.exec.physical.rowSet.DirectRowSet) SchemaBuilder(org.apache.drill.exec.record.metadata.SchemaBuilder) DirectRowSet(org.apache.drill.exec.physical.rowSet.DirectRowSet) RowSet(org.apache.drill.exec.physical.rowSet.RowSet) RowSetReader(org.apache.drill.exec.physical.rowSet.RowSetReader) Test(org.junit.Test) EvfTest(org.apache.drill.categories.EvfTest)

Example 18 with DirectRowSet

use of org.apache.drill.exec.physical.rowSet.DirectRowSet in project drill by apache.

the class TestPartitionRace method testSingleScan.

/**
 * Oddly, when run in a single fragment, the files occur in a
 * stable order, the partition always appears, and it appears in
 * the first column position.
 */
@Test
public void testSingleScan() throws IOException {
    String sql = "SELECT * FROM `dfs.data`.`%s`";
    TupleMetadata expectedSchema = new SchemaBuilder().add("a", MinorType.VARCHAR).add("b", MinorType.VARCHAR).add("c", MinorType.VARCHAR).addNullable("dir0", MinorType.VARCHAR).buildSchema();
    Iterator<DirectRowSet> iter = client.queryBuilder().sql(sql, PART_DIR).rowSetIterator();
    RowSet rowSet;
    if (SCHEMA_BATCH_ENABLED) {
        // First batch is empty; just carries the schema.
        assertTrue(iter.hasNext());
        rowSet = iter.next();
        assertEquals(0, rowSet.rowCount());
        rowSet.clear();
    }
    for (int j = 0; j < 2; j++) {
        assertTrue(iter.hasNext());
        rowSet = iter.next();
        // Figure out which record this is and test accordingly.
        RowSetReader reader = rowSet.reader();
        assertTrue(reader.next());
        String col1 = reader.scalar("a").getString();
        if (col1.equals("10")) {
            RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema).addRow("10", "foo", "bar", null).build();
            RowSetUtilities.verify(expected, rowSet);
        } else {
            RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema).addRow("20", "fred", "wilma", NESTED_DIR).build();
            RowSetUtilities.verify(expected, rowSet);
        }
    }
    assertFalse(iter.hasNext());
}
Also used : RowSetBuilder(org.apache.drill.exec.physical.rowSet.RowSetBuilder) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) SchemaBuilder(org.apache.drill.exec.record.metadata.SchemaBuilder) DirectRowSet(org.apache.drill.exec.physical.rowSet.DirectRowSet) DirectRowSet(org.apache.drill.exec.physical.rowSet.DirectRowSet) RowSet(org.apache.drill.exec.physical.rowSet.RowSet) RowSetReader(org.apache.drill.exec.physical.rowSet.RowSetReader) Test(org.junit.Test) EvfTest(org.apache.drill.categories.EvfTest)

Example 19 with DirectRowSet

use of org.apache.drill.exec.physical.rowSet.DirectRowSet in project drill by apache.

the class TestCsvWithSchema method testMultiFileSchema.

/**
 * Use a schema with explicit projection to get a consistent view
 * of the table schema, even if columns are missing, rows are ragged,
 * and column order changes.
 * <p>
 * Force the scans to occur in distinct fragments so the order of the
 * file batches is random.
 */
@Test
public void testMultiFileSchema() throws Exception {
    RowSet expected1 = null;
    RowSet expected2 = null;
    try {
        enableSchemaSupport();
        enableMultiScan();
        String tablePath = buildTable("multiFileSchema", raggedMulti1Contents, reordered2Contents);
        run(SCHEMA_SQL, tablePath);
        // Wildcard expands to union of schema + table. In this case
        // all table columns appear in the schema (though not all schema
        // columns appear in the table.)
        String sql = "SELECT id, `name`, `date`, gender, comment FROM " + tablePath;
        TupleMetadata expectedSchema = new SchemaBuilder().add("id", MinorType.INT).add("name", MinorType.VARCHAR).addNullable("date", MinorType.DATE).add("gender", MinorType.VARCHAR).add("comment", MinorType.VARCHAR).buildSchema();
        expected1 = new RowSetBuilder(client.allocator(), expectedSchema).addRow(1, "wilma", LocalDate.of(2019, 1, 18), "female", "ABC").addRow(2, "fred", LocalDate.of(2019, 1, 19), "male", "ABC").addRow(4, "betty", LocalDate.of(2019, 5, 4), "NA", "ABC").build();
        expected2 = new RowSetBuilder(client.allocator(), expectedSchema).addRow(3, "barney", LocalDate.of(2001, 1, 16), "NA", "ABC").build();
        for (int i = 0; i < 10; i++) {
            boolean sawSchema = false;
            boolean sawFile1 = false;
            boolean sawFile2 = false;
            Iterator<DirectRowSet> iter = client.queryBuilder().sql(sql).rowSetIterator();
            while (iter.hasNext()) {
                RowSet result = iter.next();
                if (result.rowCount() == 3) {
                    sawFile1 = true;
                    new RowSetComparison(expected1).verifyAndClear(result);
                } else if (result.rowCount() == 1) {
                    sawFile2 = true;
                    new RowSetComparison(expected2).verifyAndClear(result);
                } else {
                    assertEquals(0, result.rowCount());
                    sawSchema = true;
                }
            }
            assertTrue(!SCHEMA_BATCH_ENABLED || sawSchema);
            assertTrue(sawFile1);
            assertTrue(sawFile2);
        }
    } finally {
        expected1.clear();
        expected2.clear();
        resetSchemaSupport();
        resetMultiScan();
    }
}
Also used : RowSetBuilder(org.apache.drill.exec.physical.rowSet.RowSetBuilder) RowSetComparison(org.apache.drill.test.rowSet.RowSetComparison) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) DirectRowSet(org.apache.drill.exec.physical.rowSet.DirectRowSet) RowSet(org.apache.drill.exec.physical.rowSet.RowSet) SchemaBuilder(org.apache.drill.exec.record.metadata.SchemaBuilder) DirectRowSet(org.apache.drill.exec.physical.rowSet.DirectRowSet) Test(org.junit.Test) EvfTest(org.apache.drill.categories.EvfTest)

Example 20 with DirectRowSet

use of org.apache.drill.exec.physical.rowSet.DirectRowSet in project drill by apache.

the class TestCsvWithHeaders method testPartitionExpansion.

/**
 * Test partition expansion.
 * <p>
 * This test is tricky because it will return two data batches
 * (preceded by an empty schema batch.) File read order is random
 * so we have to expect the files in either order.
 * <p>
 * V3 puts partition columns after
 * data columns (so that data columns don't shift positions if
 * files are nested to another level.)
 */
@Test
public void testPartitionExpansion() {
    Iterator<DirectRowSet> iter = client.queryBuilder().sql(makeStatement(PART_DIR)).rowSetIterator();
    TupleMetadata expectedSchema = new SchemaBuilder().add("a", MinorType.VARCHAR).add("b", MinorType.VARCHAR).add("c", MinorType.VARCHAR).addNullable("dir0", MinorType.VARCHAR).buildSchema();
    RowSet rowSet;
    if (SCHEMA_BATCH_ENABLED) {
        // First batch is empty; just carries the schema.
        assertTrue(iter.hasNext());
        rowSet = iter.next();
        assertEquals(0, rowSet.rowCount());
        rowSet.clear();
    }
    // Read the other two batches.
    for (int i = 0; i < 2; i++) {
        assertTrue(iter.hasNext());
        rowSet = iter.next();
        // Figure out which record this is and test accordingly.
        RowSetReader reader = rowSet.reader();
        assertTrue(reader.next());
        String col1 = reader.scalar(0).getString();
        if (col1.equals("10")) {
            RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema).addRow("10", "foo", "bar", null).build();
            RowSetUtilities.verify(expected, rowSet);
        } else {
            RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema).addRow("20", "fred", "wilma", NESTED_DIR).build();
            RowSetUtilities.verify(expected, rowSet);
        }
    }
    assertFalse(iter.hasNext());
}
Also used : RowSetBuilder(org.apache.drill.exec.physical.rowSet.RowSetBuilder) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) DirectRowSet(org.apache.drill.exec.physical.rowSet.DirectRowSet) SchemaBuilder(org.apache.drill.exec.record.metadata.SchemaBuilder) DirectRowSet(org.apache.drill.exec.physical.rowSet.DirectRowSet) RowSet(org.apache.drill.exec.physical.rowSet.RowSet) RowSetReader(org.apache.drill.exec.physical.rowSet.RowSetReader) Test(org.junit.Test) EvfTest(org.apache.drill.categories.EvfTest)

Aggregations

DirectRowSet (org.apache.drill.exec.physical.rowSet.DirectRowSet)42 Test (org.junit.Test)40 RowSet (org.apache.drill.exec.physical.rowSet.RowSet)39 SchemaBuilder (org.apache.drill.exec.record.metadata.SchemaBuilder)39 TupleMetadata (org.apache.drill.exec.record.metadata.TupleMetadata)39 RowSetBuilder (org.apache.drill.exec.physical.rowSet.RowSetBuilder)35 ClusterTest (org.apache.drill.test.ClusterTest)33 QuerySummary (org.apache.drill.test.QueryBuilder.QuerySummary)25 JdbcStorageTest (org.apache.drill.categories.JdbcStorageTest)21 RowSetReader (org.apache.drill.exec.physical.rowSet.RowSetReader)8 EvfTest (org.apache.drill.categories.EvfTest)7 MockResponse (okhttp3.mockwebserver.MockResponse)3 MockWebServer (okhttp3.mockwebserver.MockWebServer)3 UserRemoteException (org.apache.drill.common.exceptions.UserRemoteException)3 IOException (java.io.IOException)2 Request (okhttp3.Request)2 Response (okhttp3.Response)2 PersistentTokenTable (org.apache.drill.exec.oauth.PersistentTokenTable)2 ClusterFixtureBuilder (org.apache.drill.test.ClusterFixtureBuilder)2 BigDecimal (java.math.BigDecimal)1