Search in sources :

Example 16 with VarCharVector

use of org.apache.drill.exec.vector.VarCharVector in project drill by apache.

the class TestHiveUDFs method testGenericUDF.

@Test
public void testGenericUDF() throws Throwable {
    int numRecords = 0;
    String planString = Resources.toString(Resources.getResource("functions/hive/GenericUDF.json"), Charsets.UTF_8);
    List<QueryDataBatch> results = testPhysicalWithResults(planString);
    RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());
    for (QueryDataBatch result : results) {
        batchLoader.load(result.getHeader().getDef(), result.getData());
        if (batchLoader.getRecordCount() <= 0) {
            result.release();
            batchLoader.clear();
            continue;
        }
        // Output columns and types
        //  1. str1 : VarChar
        //  2. upperStr1 : NullableVarChar
        //  3. concat : NullableVarChar
        //  4. flt1 : Float4
        //  5. format_number : NullableFloat8
        //  6. nullableStr1 : NullableVarChar
        //  7. upperNullableStr1 : NullableVarChar
        VarCharVector str1V = (VarCharVector) batchLoader.getValueAccessorById(VarCharVector.class, 0).getValueVector();
        NullableVarCharVector upperStr1V = (NullableVarCharVector) batchLoader.getValueAccessorById(NullableVarCharVector.class, 1).getValueVector();
        NullableVarCharVector concatV = (NullableVarCharVector) batchLoader.getValueAccessorById(NullableVarCharVector.class, 2).getValueVector();
        Float4Vector flt1V = (Float4Vector) batchLoader.getValueAccessorById(Float4Vector.class, 3).getValueVector();
        NullableVarCharVector format_numberV = (NullableVarCharVector) batchLoader.getValueAccessorById(NullableVarCharVector.class, 4).getValueVector();
        NullableVarCharVector nullableStr1V = (NullableVarCharVector) batchLoader.getValueAccessorById(NullableVarCharVector.class, 5).getValueVector();
        NullableVarCharVector upperNullableStr1V = (NullableVarCharVector) batchLoader.getValueAccessorById(NullableVarCharVector.class, 6).getValueVector();
        for (int i = 0; i < batchLoader.getRecordCount(); i++) {
            String in = new String(str1V.getAccessor().get(i), Charsets.UTF_8);
            String upper = new String(upperStr1V.getAccessor().get(i), Charsets.UTF_8);
            assertTrue(in.toUpperCase().equals(upper));
            String concat = new String(concatV.getAccessor().get(i), Charsets.UTF_8);
            assertTrue(concat.equals(in + "-" + in));
            float flt1 = flt1V.getAccessor().get(i);
            String format_number = new String(format_numberV.getAccessor().get(i), Charsets.UTF_8);
            String nullableStr1 = null;
            if (!nullableStr1V.getAccessor().isNull(i)) {
                nullableStr1 = new String(nullableStr1V.getAccessor().get(i), Charsets.UTF_8);
            }
            String upperNullableStr1 = null;
            if (!upperNullableStr1V.getAccessor().isNull(i)) {
                upperNullableStr1 = new String(upperNullableStr1V.getAccessor().get(i), Charsets.UTF_8);
            }
            assertEquals(nullableStr1 != null, upperNullableStr1 != null);
            if (nullableStr1 != null) {
                assertEquals(nullableStr1.toUpperCase(), upperNullableStr1);
            }
            System.out.println(in + ", " + upper + ", " + concat + ", " + flt1 + ", " + format_number + ", " + nullableStr1 + ", " + upperNullableStr1);
            numRecords++;
        }
        result.release();
        batchLoader.clear();
    }
    System.out.println("Processed " + numRecords + " records");
}
Also used : QueryDataBatch(org.apache.drill.exec.rpc.user.QueryDataBatch) NullableVarCharVector(org.apache.drill.exec.vector.NullableVarCharVector) Float4Vector(org.apache.drill.exec.vector.Float4Vector) RecordBatchLoader(org.apache.drill.exec.record.RecordBatchLoader) NullableVarCharVector(org.apache.drill.exec.vector.NullableVarCharVector) VarCharVector(org.apache.drill.exec.vector.VarCharVector) Test(org.junit.Test)

Example 17 with VarCharVector

use of org.apache.drill.exec.vector.VarCharVector in project drill by apache.

the class TestStringFunctions method getRunResult.

public Object[] getRunResult(SimpleRootExec exec) {
    int size = 0;
    for (final ValueVector v : exec) {
        size++;
    }
    final Object[] res = new Object[size];
    int i = 0;
    for (final ValueVector v : exec) {
        if (v instanceof VarCharVector) {
            res[i++] = new String(((VarCharVector) v).getAccessor().get(0), Charsets.UTF_8);
        } else {
            res[i++] = v.getAccessor().getObject(0);
        }
    }
    return res;
}
Also used : ValueVector(org.apache.drill.exec.vector.ValueVector) VarCharVector(org.apache.drill.exec.vector.VarCharVector)

Example 18 with VarCharVector

use of org.apache.drill.exec.vector.VarCharVector in project drill by apache.

the class TestBatchValidator method testVariableMissingLast.

@Test
public void testVariableMissingLast() {
    BatchSchema schema = new SchemaBuilder().add("a", MinorType.VARCHAR).build();
    SingleRowSet batch = fixture.rowSetBuilder(schema).add("x").add("y").add("z").build();
    // Here we are evil: stomp on the last offset to simulate corruption.
    // Don't do this in real code!
    VectorAccessible va = batch.vectorAccessible();
    @SuppressWarnings("resource") ValueVector v = va.iterator().next().getValueVector();
    VarCharVector vc = (VarCharVector) v;
    @SuppressWarnings("resource") UInt4Vector ov = vc.getOffsetVector();
    assertTrue(ov.getAccessor().get(3) > 0);
    ov.getMutator().set(3, 0);
    // Validator should catch the error.
    BatchValidator validator = new BatchValidator(batch.vectorAccessible(), true);
    validator.validate();
    List<String> errors = validator.errors();
    assertEquals(1, errors.size());
    assertTrue(errors.get(0).contains("Decreasing offsets"));
    batch.clear();
}
Also used : ValueVector(org.apache.drill.exec.vector.ValueVector) SingleRowSet(org.apache.drill.test.rowSet.RowSet.SingleRowSet) VectorAccessible(org.apache.drill.exec.record.VectorAccessible) BatchSchema(org.apache.drill.exec.record.BatchSchema) SchemaBuilder(org.apache.drill.test.rowSet.SchemaBuilder) RepeatedVarCharVector(org.apache.drill.exec.vector.RepeatedVarCharVector) VarCharVector(org.apache.drill.exec.vector.VarCharVector) BatchValidator(org.apache.drill.exec.physical.impl.validate.BatchValidator) UInt4Vector(org.apache.drill.exec.vector.UInt4Vector) Test(org.junit.Test)

Example 19 with VarCharVector

use of org.apache.drill.exec.vector.VarCharVector in project drill by apache.

the class TestLoad method testLoadValueVector.

@Test
public void testLoadValueVector() throws Exception {
    final BufferAllocator allocator = RootAllocatorFactory.newRoot(drillConfig);
    final ValueVector fixedV = new IntVector(MaterializedField.create("ints", Types.required(MinorType.INT)), allocator);
    final ValueVector varlenV = new VarCharVector(MaterializedField.create("chars", Types.required(MinorType.VARCHAR)), allocator);
    final ValueVector nullableVarlenV = new NullableVarCharVector(MaterializedField.create("chars", Types.optional(MinorType.VARCHAR)), allocator);
    final List<ValueVector> vectors = Lists.newArrayList(fixedV, varlenV, nullableVarlenV);
    for (final ValueVector v : vectors) {
        AllocationHelper.allocate(v, 100, 50);
        v.getMutator().generateTestData(100);
    }
    final WritableBatch writableBatch = WritableBatch.getBatchNoHV(100, vectors, false);
    final RecordBatchLoader batchLoader = new RecordBatchLoader(allocator);
    final ByteBuf[] byteBufs = writableBatch.getBuffers();
    int bytes = 0;
    for (int i = 0; i < byteBufs.length; i++) {
        bytes += byteBufs[i].writerIndex();
    }
    final DrillBuf byteBuf = allocator.buffer(bytes);
    int index = 0;
    for (int i = 0; i < byteBufs.length; i++) {
        byteBufs[i].readBytes(byteBuf, index, byteBufs[i].writerIndex());
        index += byteBufs[i].writerIndex();
    }
    byteBuf.writerIndex(bytes);
    batchLoader.load(writableBatch.getDef(), byteBuf);
    boolean firstColumn = true;
    int recordCount = 0;
    for (final VectorWrapper<?> v : batchLoader) {
        if (firstColumn) {
            firstColumn = false;
        } else {
            System.out.print("\t");
        }
        System.out.print(v.getField().getPath());
        System.out.print("[");
        System.out.print(v.getField().getType().getMinorType());
        System.out.print("]");
    }
    System.out.println();
    for (int r = 0; r < batchLoader.getRecordCount(); r++) {
        boolean first = true;
        recordCount++;
        for (final VectorWrapper<?> v : batchLoader) {
            if (first) {
                first = false;
            } else {
                System.out.print("\t");
            }
            final ValueVector.Accessor accessor = v.getValueVector().getAccessor();
            if (v.getField().getType().getMinorType() == TypeProtos.MinorType.VARCHAR) {
                final Object obj = accessor.getObject(r);
                if (obj != null) {
                    System.out.print(accessor.getObject(r));
                } else {
                    System.out.print("NULL");
                }
            } else {
                System.out.print(accessor.getObject(r));
            }
        }
        if (!first) {
            System.out.println();
        }
    }
    assertEquals(100, recordCount);
    batchLoader.clear();
    writableBatch.clear();
}
Also used : IntVector(org.apache.drill.exec.vector.IntVector) RecordBatchLoader(org.apache.drill.exec.record.RecordBatchLoader) NullableVarCharVector(org.apache.drill.exec.vector.NullableVarCharVector) VarCharVector(org.apache.drill.exec.vector.VarCharVector) ByteBuf(io.netty.buffer.ByteBuf) BufferAllocator(org.apache.drill.exec.memory.BufferAllocator) ValueVector(org.apache.drill.exec.vector.ValueVector) NullableVarCharVector(org.apache.drill.exec.vector.NullableVarCharVector) WritableBatch(org.apache.drill.exec.record.WritableBatch) DrillBuf(io.netty.buffer.DrillBuf) Test(org.junit.Test) ExecTest(org.apache.drill.exec.ExecTest)

Example 20 with VarCharVector

use of org.apache.drill.exec.vector.VarCharVector in project drill by apache.

the class TestNewMathFunctions method getRunResult.

public Object[] getRunResult(SimpleRootExec exec) {
    int size = 0;
    for (final ValueVector v : exec) {
        size++;
    }
    final Object[] res = new Object[size];
    int i = 0;
    for (final ValueVector v : exec) {
        if (v instanceof VarCharVector) {
            res[i++] = new String(((VarCharVector) v).getAccessor().get(0));
        } else {
            res[i++] = v.getAccessor().getObject(0);
        }
    }
    return res;
}
Also used : ValueVector(org.apache.drill.exec.vector.ValueVector) VarCharVector(org.apache.drill.exec.vector.VarCharVector)

Aggregations

VarCharVector (org.apache.drill.exec.vector.VarCharVector)20 ValueVector (org.apache.drill.exec.vector.ValueVector)9 Test (org.junit.Test)9 NullableVarCharVector (org.apache.drill.exec.vector.NullableVarCharVector)7 RecordBatchLoader (org.apache.drill.exec.record.RecordBatchLoader)6 QueryDataBatch (org.apache.drill.exec.rpc.user.QueryDataBatch)5 RepeatedVarCharVector (org.apache.drill.exec.vector.RepeatedVarCharVector)4 UInt4Vector (org.apache.drill.exec.vector.UInt4Vector)4 ExecTest (org.apache.drill.exec.ExecTest)3 MaterializedField (org.apache.drill.exec.record.MaterializedField)3 VectorAccessible (org.apache.drill.exec.record.VectorAccessible)3 BigIntVector (org.apache.drill.exec.vector.BigIntVector)3 DrillBuf (io.netty.buffer.DrillBuf)2 ArrayList (java.util.ArrayList)2 BatchValidator (org.apache.drill.exec.physical.impl.validate.BatchValidator)2 BatchSchema (org.apache.drill.exec.record.BatchSchema)2 Float4Vector (org.apache.drill.exec.vector.Float4Vector)2 IntVector (org.apache.drill.exec.vector.IntVector)2 SingleRowSet (org.apache.drill.test.rowSet.RowSet.SingleRowSet)2 SchemaBuilder (org.apache.drill.test.rowSet.SchemaBuilder)2