Examples with NullableVarCharVector org.apache.drill.exec.vector.NullableVarCharVector used on opensource projects

Example 11 with NullableVarCharVector

use of org.apache.drill.exec.vector.NullableVarCharVector in project drill by apache.

the class TestResultSetLoaderUnions method testVariantListDynamic.

/**
 * Test a variant list created dynamically at load time.
 * The list starts with no type, at which time it can hold
 * only null values. Then we add a Varchar, and finally an
 * Int.
 * <p>
 * This test is superficial. There are many odd cases to consider.
 * <ul>
 * <li>Write nulls to a list with no type. (This test ensures that
 * adding a (nullable) scalar "does the right thing."</li>
 * <li>Add a map to the list. Maps carry no "bits" vector, so null
 * list entries to that point are lost. (For maps, we could go straight
 * to a union, with just a map, to preserve the null states. This whole
 * area is a huge mess...)</li>
 * <li>Do the type transitions when writing to a row. (The tests here
 * do the transition between rows.)</li>
 * </ul>
 *
 * The reason for the sparse coverage is that Drill barely supports lists
 * and unions; most code is just plain broken. Our goal here is not to fix
 * all those problems, just to leave things no more broken than before.
 */
@Test
public void testVariantListDynamic() {
    final ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator());
    final RowSetLoader writer = rsLoader.writer();
    // Can write a batch as if this was a repeated Varchar, except
    // that any value can also be null.
    rsLoader.startBatch();
    writer.addColumn(MaterializedField.create("id", Types.required(MinorType.INT)));
    writer.addColumn(MaterializedField.create("list", Types.optional(MinorType.LIST)));
    // Sanity check: should be an array of variants because we said the
    // types within the list are expandable (which is the default.)
    final ArrayWriter arrWriter = writer.array("list");
    assertEquals(ObjectType.VARIANT, arrWriter.entryType());
    final VariantWriter variant = arrWriter.variant();
    // We need to verify that the internal state is what we expect, so
    // the next assertion peeks inside the private bits of the union
    // writer. No client code should ever need to do this, of course.
    assertTrue(((UnionWriterImpl) variant).shim() instanceof EmptyListShim);
    // No types, so all we can do is add a null list, or a list of nulls.
    writer.addRow(1, null).addRow(2, variantArray()).addRow(3, variantArray(null, null));
    // Add a String. Now we can create a list of strings and/or nulls.
    variant.addMember(MinorType.VARCHAR);
    assertTrue(variant.hasType(MinorType.VARCHAR));
    // Sanity check: sniff inside to ensure that the list contains a single
    // type.
    assertTrue(((UnionWriterImpl) variant).shim() instanceof SimpleListShim);
    assertTrue(((ListWriterImpl) arrWriter).vector().getDataVector() instanceof NullableVarCharVector);
    writer.addRow(4, variantArray("fred", null, "barney"));
    // Add an integer. The list vector should be promoted to union.
    // Now we can add both types.
    variant.addMember(MinorType.INT);
    // Sanity check: sniff inside to ensure promotion to union occurred
    assertTrue(((UnionWriterImpl) variant).shim() instanceof UnionVectorShim);
    assertTrue(((ListWriterImpl) arrWriter).vector().getDataVector() instanceof UnionVector);
    writer.addRow(5, variantArray("wilma", null, 30));
    // Verify
    final RowSet result = fixture.wrap(rsLoader.harvest());
    final TupleMetadata schema = new SchemaBuilder().add("id", MinorType.INT).addList("list").addType(MinorType.VARCHAR).addType(MinorType.INT).resumeSchema().buildSchema();
    final SingleRowSet expected = fixture.rowSetBuilder(schema).addRow(1, null).addRow(2, variantArray()).addRow(3, variantArray(null, null)).addRow(4, variantArray("fred", null, "barney")).addRow(5, variantArray("wilma", null, 30)).build();
    RowSetUtilities.verify(expected, result);
}

Example 12 with NullableVarCharVector

use of org.apache.drill.exec.vector.NullableVarCharVector in project drill by apache.

the class ParquetPartitionDescriptor method populatePruningVector.

private void populatePruningVector(ValueVector v, int index, SchemaPath column, Path file) {
    Path path = Path.getPathWithoutSchemeAndAuthority(file);
    TypeProtos.MajorType majorType = getVectorType(column, null);
    TypeProtos.MinorType type = majorType.getMinorType();
    switch(type) {
        case BIT:
            {
                NullableBitVector bitVector = (NullableBitVector) v;
                Boolean value = groupScan.getPartitionValue(path, column, Boolean.class);
                if (value == null) {
                    bitVector.getMutator().setNull(index);
                } else {
                    bitVector.getMutator().setSafe(index, value ? 1 : 0);
                }
                return;
            }
        case INT:
            {
                NullableIntVector intVector = (NullableIntVector) v;
                Integer value = groupScan.getPartitionValue(path, column, Integer.class);
                if (value == null) {
                    intVector.getMutator().setNull(index);
                } else {
                    intVector.getMutator().setSafe(index, value);
                }
                return;
            }
        case SMALLINT:
            {
                NullableSmallIntVector smallIntVector = (NullableSmallIntVector) v;
                Integer value = groupScan.getPartitionValue(path, column, Integer.class);
                if (value == null) {
                    smallIntVector.getMutator().setNull(index);
                } else {
                    smallIntVector.getMutator().setSafe(index, value.shortValue());
                }
                return;
            }
        case TINYINT:
            {
                NullableTinyIntVector tinyIntVector = (NullableTinyIntVector) v;
                Integer value = groupScan.getPartitionValue(path, column, Integer.class);
                if (value == null) {
                    tinyIntVector.getMutator().setNull(index);
                } else {
                    tinyIntVector.getMutator().setSafe(index, value.byteValue());
                }
                return;
            }
        case UINT1:
            {
                NullableUInt1Vector intVector = (NullableUInt1Vector) v;
                Integer value = groupScan.getPartitionValue(path, column, Integer.class);
                if (value == null) {
                    intVector.getMutator().setNull(index);
                } else {
                    intVector.getMutator().setSafe(index, value.byteValue());
                }
                return;
            }
        case UINT2:
            {
                NullableUInt2Vector intVector = (NullableUInt2Vector) v;
                Integer value = groupScan.getPartitionValue(path, column, Integer.class);
                if (value == null) {
                    intVector.getMutator().setNull(index);
                } else {
                    intVector.getMutator().setSafe(index, (char) value.shortValue());
                }
                return;
            }
        case UINT4:
            {
                NullableUInt4Vector intVector = (NullableUInt4Vector) v;
                Integer value = groupScan.getPartitionValue(path, column, Integer.class);
                if (value == null) {
                    intVector.getMutator().setNull(index);
                } else {
                    intVector.getMutator().setSafe(index, value);
                }
                return;
            }
        case BIGINT:
            {
                NullableBigIntVector bigIntVector = (NullableBigIntVector) v;
                Long value = groupScan.getPartitionValue(path, column, Long.class);
                if (value == null) {
                    bigIntVector.getMutator().setNull(index);
                } else {
                    bigIntVector.getMutator().setSafe(index, value);
                }
                return;
            }
        case FLOAT4:
            {
                NullableFloat4Vector float4Vector = (NullableFloat4Vector) v;
                Float value = groupScan.getPartitionValue(path, column, Float.class);
                if (value == null) {
                    float4Vector.getMutator().setNull(index);
                } else {
                    float4Vector.getMutator().setSafe(index, value);
                }
                return;
            }
        case FLOAT8:
            {
                NullableFloat8Vector float8Vector = (NullableFloat8Vector) v;
                Double value = groupScan.getPartitionValue(path, column, Double.class);
                if (value == null) {
                    float8Vector.getMutator().setNull(index);
                } else {
                    float8Vector.getMutator().setSafe(index, value);
                }
                return;
            }
        case VARBINARY:
            {
                NullableVarBinaryVector varBinaryVector = (NullableVarBinaryVector) v;
                Object s = groupScan.getPartitionValue(path, column, Object.class);
                byte[] bytes;
                if (s == null) {
                    varBinaryVector.getMutator().setNull(index);
                    return;
                } else {
                    bytes = getBytes(type, s);
                }
                varBinaryVector.getMutator().setSafe(index, bytes, 0, bytes.length);
                return;
            }
        case VARDECIMAL:
            {
                NullableVarDecimalVector decimalVector = (NullableVarDecimalVector) v;
                Object s = groupScan.getPartitionValue(path, column, Object.class);
                byte[] bytes;
                if (s == null) {
                    decimalVector.getMutator().setNull(index);
                    return;
                } else if (s instanceof Integer) {
                    bytes = Ints.toByteArray((int) s);
                } else if (s instanceof Long) {
                    bytes = Longs.toByteArray((long) s);
                } else {
                    bytes = getBytes(type, s);
                }
                decimalVector.getMutator().setSafe(index, bytes, 0, bytes.length);
                return;
            }
        case DATE:
            {
                NullableDateVector dateVector = (NullableDateVector) v;
                Long value = groupScan.getPartitionValue(path, column, Long.class);
                if (value == null) {
                    dateVector.getMutator().setNull(index);
                } else {
                    dateVector.getMutator().setSafe(index, value);
                }
                return;
            }
        case TIME:
            {
                NullableTimeVector timeVector = (NullableTimeVector) v;
                Integer value = groupScan.getPartitionValue(path, column, Integer.class);
                if (value == null) {
                    timeVector.getMutator().setNull(index);
                } else {
                    timeVector.getMutator().setSafe(index, value);
                }
                return;
            }
        case TIMESTAMP:
            {
                NullableTimeStampVector timeStampVector = (NullableTimeStampVector) v;
                Long value = groupScan.getPartitionValue(path, column, Long.class);
                if (value == null) {
                    timeStampVector.getMutator().setNull(index);
                } else {
                    timeStampVector.getMutator().setSafe(index, value);
                }
                return;
            }
        case VARCHAR:
            {
                NullableVarCharVector varCharVector = (NullableVarCharVector) v;
                Object s = groupScan.getPartitionValue(path, column, Object.class);
                byte[] bytes;
                if (s == null) {
                    varCharVector.getMutator().setNull(index);
                    return;
                } else {
                    bytes = getBytes(type, s);
                }
                varCharVector.getMutator().setSafe(index, bytes, 0, bytes.length);
                return;
            }
        case INTERVAL:
            {
                NullableIntervalVector intervalVector = (NullableIntervalVector) v;
                Object s = groupScan.getPartitionValue(path, column, Object.class);
                byte[] bytes;
                if (s == null) {
                    intervalVector.getMutator().setNull(index);
                    return;
                } else {
                    bytes = getBytes(type, s);
                }
                intervalVector.getMutator().setSafe(index, 1, ParquetReaderUtility.getIntFromLEBytes(bytes, 0), ParquetReaderUtility.getIntFromLEBytes(bytes, 4), ParquetReaderUtility.getIntFromLEBytes(bytes, 8));
                return;
            }
        default:
            throw new UnsupportedOperationException("Unsupported type: " + type);
    }
}

Example 13 with NullableVarCharVector

use of org.apache.drill.exec.vector.NullableVarCharVector in project drill by apache.

the class TestValueVector method testNullableVarLen2.

@Test
public void testNullableVarLen2() {
    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
    // Create a new value vector for 1024 integers.
    try (final NullableVarCharVector vector = new NullableVarCharVector(field, allocator)) {
        final NullableVarCharVector.Mutator m = vector.getMutator();
        vector.allocateNew(1024 * 10, 1024);
        m.set(0, STR1);
        m.set(1, STR2);
        m.set(2, STR3);
        // Check the sample strings.
        final NullableVarCharVector.Accessor accessor = vector.getAccessor();
        assertArrayEquals(STR1, accessor.get(0));
        assertArrayEquals(STR2, accessor.get(1));
        assertArrayEquals(STR3, accessor.get(2));
        // Ensure null value throws.
        boolean b = false;
        try {
            vector.getAccessor().get(3);
        } catch (IllegalStateException e) {
            b = true;
        } finally {
            assertTrue(b);
        }
    }
}

Example 14 with NullableVarCharVector

use of org.apache.drill.exec.vector.NullableVarCharVector in project drill by apache.

the class TestValueVector method testVectors.

/**
 * Convenience method that allows running tests on various {@link ValueVector vector} instances.
 *
 * @param test test function to execute
 */
private void testVectors(VectorVerifier test) throws Exception {
    final MaterializedField[] fields = { MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE), MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE), MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE), MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE), MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedListVector.TYPE), MaterializedField.create(EMPTY_SCHEMA_PATH, MapVector.TYPE), MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedMapVector.TYPE) };
    final ValueVector[] vectors = { new UInt4Vector(fields[0], allocator), new BitVector(fields[1], allocator), new VarCharVector(fields[2], allocator), new NullableVarCharVector(fields[3], allocator), new RepeatedListVector(fields[4], allocator, null), new MapVector(fields[5], allocator, null), new RepeatedMapVector(fields[6], allocator, null) };
    try {
        for (final ValueVector vector : vectors) {
            test.verify(vector);
        }
    } finally {
        AutoCloseables.close(vectors);
    }
}

Example 15 with NullableVarCharVector

use of org.apache.drill.exec.vector.NullableVarCharVector in project drill by apache.

the class TestValueVector method testNullableVarCharVectorLoad.

@Test
public void testNullableVarCharVectorLoad() {
    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
    // Create a new value vector for 1024 nullable variable length strings.
    final NullableVarCharVector vector1 = new NullableVarCharVector(field, allocator);
    final NullableVarCharVector.Mutator mutator = vector1.getMutator();
    vector1.allocateNew(1024 * 10, 1024);
    // Populate the vector.
    final StringBuilder stringBuilder = new StringBuilder();
    final int valueCount = 10;
    for (int i = 0; i < valueCount; ++i) {
        stringBuilder.append('x');
        mutator.set(i, stringBuilder.toString().getBytes(utf8Charset));
    }
    // Check the contents.
    final NullableVarCharVector.Accessor accessor1 = vector1.getAccessor();
    stringBuilder.setLength(0);
    for (int i = 0; i < valueCount; ++i) {
        stringBuilder.append('x');
        final Object object = accessor1.getObject(i);
        assertEquals(stringBuilder.toString(), object.toString());
    }
    mutator.setValueCount(valueCount);
    assertEquals(valueCount, vector1.getAccessor().getValueCount());
    // Still ok after setting value count?
    stringBuilder.setLength(0);
    for (int i = 0; i < valueCount; ++i) {
        stringBuilder.append('x');
        final Object object = accessor1.getObject(i);
        assertEquals(stringBuilder.toString(), object.toString());
    }
    // Combine into a single buffer so we can load it into a new vector.
    final DrillBuf[] buffers1 = vector1.getBuffers(false);
    final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
    final NullableVarCharVector vector2 = new NullableVarCharVector(field, allocator);
    vector2.load(vector1.getMetadata(), buffer1);
    // Check the vector's contents.
    final NullableVarCharVector.Accessor accessor2 = vector2.getAccessor();
    stringBuilder.setLength(0);
    for (int i = 0; i < valueCount; ++i) {
        stringBuilder.append('x');
        final Object object = accessor2.getObject(i);
        assertEquals(stringBuilder.toString(), object.toString());
    }
    vector1.close();
    vector2.close();
    buffer1.release();
}