Examples with ArrayReader org.apache.drill.exec.vector.accessor.ArrayReader used on opensource projects

Example 6 with ArrayReader

use of org.apache.drill.exec.vector.accessor.ArrayReader in project drill by apache.

the class TestResultSetLoaderMapArray method testDoubleNestedArray.

/**
 * Test a doubly-nested array of maps.
 */
@Test
public void testDoubleNestedArray() {
    TupleMetadata schema = new SchemaBuilder().add("a", MinorType.INT).addMapArray("m1").add("b", MinorType.INT).addMapArray("m2").add("c", MinorType.INT).addArray("d", MinorType.VARCHAR).resumeMap().resumeSchema().buildSchema();
    ResultSetLoaderImpl.ResultSetOptions options = new ResultSetOptionBuilder().readerSchema(schema).build();
    ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
    RowSetLoader rootWriter = rsLoader.writer();
    rsLoader.startBatch();
    ScalarWriter aWriter = rootWriter.scalar("a");
    ArrayWriter a1Writer = rootWriter.array("m1");
    TupleWriter m1Writer = a1Writer.tuple();
    ScalarWriter bWriter = m1Writer.scalar("b");
    ArrayWriter a2Writer = m1Writer.array("m2");
    TupleWriter m2Writer = a2Writer.tuple();
    ScalarWriter cWriter = m2Writer.scalar("c");
    ScalarWriter dWriter = m2Writer.array("d").scalar();
    for (int i = 0; i < 5; i++) {
        rootWriter.start();
        aWriter.setInt(i);
        for (int j = 0; j < 4; j++) {
            int a1Key = i + 10 + j;
            bWriter.setInt(a1Key);
            for (int k = 0; k < 3; k++) {
                int a2Key = a1Key * 10 + k;
                cWriter.setInt(a2Key);
                for (int l = 0; l < 2; l++) {
                    dWriter.setString("d-" + (a2Key * 10 + l));
                }
                a2Writer.save();
            }
            a1Writer.save();
        }
        rootWriter.save();
    }
    RowSet results = fixture.wrap(rsLoader.harvest());
    RowSetReader reader = results.reader();
    ScalarReader aReader = reader.scalar("a");
    ArrayReader a1Reader = reader.array("m1");
    TupleReader m1Reader = a1Reader.tuple();
    ScalarReader bReader = m1Reader.scalar("b");
    ArrayReader a2Reader = m1Reader.array("m2");
    TupleReader m2Reader = a2Reader.tuple();
    ScalarReader cReader = m2Reader.scalar("c");
    ArrayReader dArray = m2Reader.array("d");
    ScalarReader dReader = dArray.scalar();
    for (int i = 0; i < 5; i++) {
        assertTrue(reader.next());
        assertEquals(i, aReader.getInt());
        for (int j = 0; j < 4; j++) {
            assertTrue(a1Reader.next());
            int a1Key = i + 10 + j;
            assertEquals(a1Key, bReader.getInt());
            for (int k = 0; k < 3; k++) {
                assertTrue(a2Reader.next());
                int a2Key = a1Key * 10 + k;
                assertEquals(a2Key, cReader.getInt());
                for (int l = 0; l < 2; l++) {
                    assertTrue(dArray.next());
                    assertEquals("d-" + (a2Key * 10 + l), dReader.getString());
                }
            }
        }
    }
    rsLoader.close();
}

Example 7 with ArrayReader

use of org.apache.drill.exec.vector.accessor.ArrayReader in project drill by apache.

the class TestFillEmpties method dofillEmptiesRepeated.

private void dofillEmptiesRepeated(MajorType majorType) {
    TupleMetadata schema = new SchemaBuilder().add("a", majorType).buildSchema();
    ExtendableRowSet rs = fixture.rowSet(schema);
    RowSetWriter writer = rs.writer();
    ScalarWriter colWriter = writer.array(0).scalar();
    ValueType valueType = colWriter.valueType();
    for (int i = 0; i < ROW_COUNT; i++) {
        if (i % 5 == 0) {
            // Write two values so we can exercise a bit of the array logic.
            colWriter.setObject(RowSetUtilities.testDataFromInt(valueType, majorType, i));
            colWriter.setObject(RowSetUtilities.testDataFromInt(valueType, majorType, i + 1));
        }
        writer.save();
    }
    SingleRowSet result = writer.done();
    RowSetReader reader = result.reader();
    ArrayReader aReader = reader.array(0);
    ScalarReader colReader = aReader.scalar();
    for (int i = 0; i < ROW_COUNT; i++) {
        assertTrue(reader.next());
        if (i % 5 != 0) {
            // Empty arrays are defined to be the same as a zero-length array.
            assertEquals(0, aReader.size());
        } else {
            for (int j = 0; j < 2; j++) {
                assertTrue(aReader.next());
                Object actual = colReader.getObject();
                Object expected = RowSetUtilities.testDataFromInt(valueType, majorType, i + j);
                RowSetUtilities.assertEqualValues(majorType.toString().replace('\n', ' ') + "[" + i + "][" + j + "]", valueType, expected, actual);
            }
        }
    }
    result.clear();
}

Example 8 with ArrayReader

use of org.apache.drill.exec.vector.accessor.ArrayReader in project drill by apache.

the class TestCsvWithoutHeaders method testHugeColumn.

@Test
public void testHugeColumn() throws IOException {
    String fileName = buildBigColFile(false);
    String sql = "SELECT * FROM `dfs.data`.`%s`";
    RowSet actual = client.queryBuilder().sql(sql, fileName).rowSet();
    assertEquals(10, actual.rowCount());
    RowSetReader reader = actual.reader();
    ArrayReader arrayReader = reader.array(0);
    while (reader.next()) {
        int i = reader.logicalIndex();
        arrayReader.next();
        assertEquals(Integer.toString(i + 1), arrayReader.scalar().getString());
        arrayReader.next();
        String big = arrayReader.scalar().getString();
        assertEquals(BIG_COL_SIZE, big.length());
        for (int j = 0; j < BIG_COL_SIZE; j++) {
            assertEquals((char) ((j + i) % 26 + 'A'), big.charAt(j));
        }
        arrayReader.next();
        assertEquals(Integer.toString((i + 1) * 10), arrayReader.scalar().getString());
    }
    actual.clear();
}

Example 9 with ArrayReader

use of org.apache.drill.exec.vector.accessor.ArrayReader in project drill by apache.

the class TestCsvWithoutHeaders method testPartitionExpansion.

/**
 * Test partition expansion.
 * <p>
 * V3, as in V2 before Drill 1.12, puts partition columns after
 * data columns (so that data columns don't shift positions if
 * files are nested to another level.)
 */
@Test
public void testPartitionExpansion() throws IOException {
    String sql = "SELECT * FROM `dfs.data`.`%s`";
    Iterator<DirectRowSet> iter = client.queryBuilder().sql(sql, PART_DIR).rowSetIterator();
    TupleMetadata expectedSchema = new SchemaBuilder().addArray("columns", MinorType.VARCHAR).addNullable("dir0", MinorType.VARCHAR).buildSchema();
    RowSet rowSet;
    if (SCHEMA_BATCH_ENABLED) {
        // First batch is empty; just carries the schema.
        assertTrue(iter.hasNext());
        rowSet = iter.next();
        assertEquals(0, rowSet.rowCount());
        rowSet.clear();
    }
    // Read the two data batches.
    for (int i = 0; i < 2; i++) {
        assertTrue(iter.hasNext());
        rowSet = iter.next();
        // Figure out which record this is and test accordingly.
        RowSetReader reader = rowSet.reader();
        assertTrue(reader.next());
        ArrayReader ar = reader.array(0);
        assertTrue(ar.next());
        String col1 = ar.scalar().getString();
        if (col1.equals("10")) {
            RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema).addRow(strArray("10", "foo", "bar"), null).addRow(strArray("20", "fred", "wilma"), null).build();
            RowSetUtilities.verify(expected, rowSet);
        } else {
            RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema).addRow(strArray("30", "barney", "betty"), NESTED_DIR).build();
            RowSetUtilities.verify(expected, rowSet);
        }
    }
    assertFalse(iter.hasNext());
}

Example 10 with ArrayReader

use of org.apache.drill.exec.vector.accessor.ArrayReader in project drill by apache.

the class MetadataControllerBatch method getIncomingLocations.

private Set<Path> getIncomingLocations(TupleReader reader) {
    Set<Path> childLocations = new HashSet<>();
    ObjectReader metadataColumnReader = reader.column(MetastoreAnalyzeConstants.METADATA_TYPE);
    Preconditions.checkNotNull(metadataColumnReader, "metadataType column wasn't found");
    MetadataType metadataType = MetadataType.valueOf(metadataColumnReader.scalar().getString());
    switch(metadataType) {
        case SEGMENT:
        case PARTITION:
            {
                ObjectReader locationsReader = reader.column(MetastoreAnalyzeConstants.LOCATIONS_FIELD);
                // populate list of file locations from "locations" field if it is present in the schema
                if (locationsReader != null && locationsReader.type() == ObjectType.ARRAY) {
                    ArrayReader array = locationsReader.array();
                    while (array.next()) {
                        childLocations.add(new Path(array.scalar().getString()));
                    }
                    break;
                }
                // in the opposite case, populate list of file locations using underlying metadata
                ObjectReader underlyingMetadataReader = reader.column(MetastoreAnalyzeConstants.COLLECTED_MAP_FIELD);
                if (underlyingMetadataReader != null) {
                    // current row contains information about underlying metadata
                    ArrayReader array = underlyingMetadataReader.array();
                    array.rewind();
                    while (array.next()) {
                        childLocations.addAll(getIncomingLocations(array.tuple()));
                    }
                }
                break;
            }
        case FILE:
            {
                childLocations.add(new Path(reader.column(MetastoreAnalyzeConstants.LOCATION_FIELD).scalar().getString()));
            }
        default:
            break;
    }
    return childLocations;
}