Examples with FieldReader org.apache.drill.exec.vector.complex.reader.FieldReader used on opensource projects

Example 11 with FieldReader

use of org.apache.drill.exec.vector.complex.reader.FieldReader in project drill by apache.

the class MappifyUtility method mappify.

public static DrillBuf mappify(FieldReader reader, BaseWriter.ComplexWriter writer, DrillBuf buffer, String caller) {
    // Currently we expect single map or null as input
    if (reader.getType().getMode() == DataMode.REPEATED || (reader.isSet() && reader.getType().getMinorType() != TypeProtos.MinorType.MAP)) {
        throw new DrillRuntimeException("kvgen function only supports Simple maps as input");
    }
    BaseWriter.ListWriter listWriter = writer.rootAsList();
    listWriter.startList();
    BaseWriter.MapWriter mapWriter = listWriter.map();
    if (!reader.isSet()) {
        // Return empty list
        listWriter.endList();
        return buffer;
    }
    // Iterate over the fields in the map
    Iterator<String> fieldIterator = reader.iterator();
    while (fieldIterator.hasNext()) {
        String str = fieldIterator.next();
        FieldReader fieldReader = reader.reader(str);
        // Skip the field if its null
        if (!fieldReader.isSet()) {
            mapWriter.end();
            continue;
        }
        // writing a new field, start a new map
        mapWriter.start();
        // write "key":"columnname" into the map
        VarCharHolder vh = new VarCharHolder();
        byte[] b = str.getBytes(Charsets.UTF_8);
        buffer = buffer.reallocIfNeeded(b.length);
        buffer.setBytes(0, b);
        vh.start = 0;
        vh.end = b.length;
        vh.buffer = buffer;
        mapWriter.varChar(fieldKey).write(vh);
        // Write the value to the map
        MapUtility.writeToMapFromReader(fieldReader, mapWriter, caller);
        mapWriter.end();
    }
    listWriter.endList();
    return buffer;
}

Example 12 with FieldReader

use of org.apache.drill.exec.vector.complex.reader.FieldReader in project drill by apache.

the class MappifyUtility method createMap.

public static void createMap(FieldReader reader, BaseWriter.ComplexWriter writer, String caller) {
    if (DataMode.REPEATED == reader.getType().getMode()) {
        throw new DrillRuntimeException("Do not invoke createMap() with REPEATED MINOR mode");
    }
    if (reader.getType().getMinorType() == TypeProtos.MinorType.MAP) {
        BaseWriter.MapWriter mapWriter = writer.rootAsMap();
        // Iterate over the fields in the map
        Iterator<String> fieldIterator = reader.iterator();
        while (fieldIterator.hasNext()) {
            String field = fieldIterator.next();
            FieldReader fieldReader = reader.reader(field);
            // Write the value to the map
            MapUtility.writeToMapFromReader(fieldReader, mapWriter, field, caller);
        }
    }
}

Example 13 with FieldReader

use of org.apache.drill.exec.vector.complex.reader.FieldReader in project drill by apache.

the class TestRepeated method listOfList.

// 
// @Test
// public void repeatedMap() {
// 
// /**
// * We're going to try to create an object that looks like:
// *
// *  {
// *    a: [
// *      {x: 1, y: 2}
// *      {x: 2, y: 1}
// *    ]
// *  }
// *
// */
// MapVector v = new MapVector("", allocator);
// ComplexWriter writer = new ComplexWriterImpl("col", v);
// 
// MapWriter map = writer.rootAsMap();
// 
// map.start();
// ListWriter list = map.list("a");
// MapWriter inner = list.map();
// 
// IntHolder holder = new IntHolder();
// IntWriter xCol = inner.integer("x");
// IntWriter yCol = inner.integer("y");
// 
// inner.start();
// 
// holder.value = 1;
// xCol.write(holder);
// holder.value = 2;
// yCol.write(holder);
// 
// inner.end();
// 
// inner.start();
// 
// holder.value = 2;
// xCol.write(holder);
// holder.value = 1;
// yCol.write(holder);
// 
// inner.end();
// 
// IntWriter numCol = map.integer("nums");
// holder.value = 14;
// numCol.write(holder);
// 
// map.end();
// 
// 
// assertTrue(writer.ok());
// 
// }
@Test
public void listOfList() throws Exception {
    /**
     * We're going to try to create an object that looks like:
     *
     *  {
     *    a: [
     *      [1,2,3],
     *      [2,3,4]
     *    ],
     *    nums: 14,
     *    b: [
     *      { c: 1 },
     *      { c: 2 , x: 15}
     *    ]
     *  }
     */
    final MapVector mapVector = new MapVector("", allocator, null);
    final ComplexWriterImpl writer = new ComplexWriterImpl("col", mapVector);
    writer.allocate();
    {
        final MapWriter map = writer.rootAsMap();
        final ListWriter list = map.list("a");
        list.startList();
        final ListWriter innerList = list.list();
        final IntWriter innerInt = innerList.integer();
        innerList.startList();
        final IntHolder holder = new IntHolder();
        holder.value = 1;
        innerInt.write(holder);
        holder.value = 2;
        innerInt.write(holder);
        holder.value = 3;
        innerInt.write(holder);
        innerList.endList();
        innerList.startList();
        holder.value = 4;
        innerInt.write(holder);
        holder.value = 5;
        innerInt.write(holder);
        innerList.endList();
        list.endList();
        final IntWriter numCol = map.integer("nums");
        holder.value = 14;
        numCol.write(holder);
        final MapWriter repeatedMap = map.list("b").map();
        repeatedMap.start();
        holder.value = 1;
        repeatedMap.integer("c").write(holder);
        repeatedMap.end();
        repeatedMap.start();
        holder.value = 2;
        repeatedMap.integer("c").write(holder);
        final BigIntHolder h = new BigIntHolder();
        h.value = 15;
        repeatedMap.bigInt("x").write(h);
        repeatedMap.end();
        map.end();
    }
    {
        writer.setPosition(1);
        final MapWriter map = writer.rootAsMap();
        final ListWriter list = map.list("a");
        list.startList();
        final ListWriter innerList = list.list();
        final IntWriter innerInt = innerList.integer();
        innerList.startList();
        final IntHolder holder = new IntHolder();
        holder.value = -1;
        innerInt.write(holder);
        holder.value = -2;
        innerInt.write(holder);
        holder.value = -3;
        innerInt.write(holder);
        innerList.endList();
        innerList.startList();
        holder.value = -4;
        innerInt.write(holder);
        holder.value = -5;
        innerInt.write(holder);
        innerList.endList();
        list.endList();
        final IntWriter numCol = map.integer("nums");
        holder.value = -28;
        numCol.write(holder);
        final MapWriter repeatedMap = map.list("b").map();
        repeatedMap.start();
        holder.value = -1;
        repeatedMap.integer("c").write(holder);
        repeatedMap.end();
        repeatedMap.start();
        holder.value = -2;
        repeatedMap.integer("c").write(holder);
        final BigIntHolder h = new BigIntHolder();
        h.value = -30;
        repeatedMap.bigInt("x").write(h);
        repeatedMap.end();
        map.end();
    }
    final ObjectWriter ow = new ObjectMapper().writer().withDefaultPrettyPrinter();
    final ByteArrayOutputStream stream = new ByteArrayOutputStream();
    final JsonWriter jsonWriter = new JsonWriter(stream, true, true);
    final FieldReader reader = mapVector.getChild("col", MapVector.class).getReader();
    reader.setPosition(0);
    jsonWriter.write(reader);
    reader.setPosition(1);
    jsonWriter.write(reader);
    writer.close();
}

Example 14 with FieldReader

use of org.apache.drill.exec.vector.complex.reader.FieldReader in project drill by apache.

the class RepeatedMapReaderImpl method reader.

@Override
public FieldReader reader(String name) {
    FieldReader reader = fields.get(name);
    if (reader == null) {
        ValueVector child = vector.getChild(name);
        if (child == null) {
            reader = NullReader.INSTANCE;
        } else {
            reader = child.getReader();
        }
        fields.put(name, reader);
        reader.setPosition(currentOffset);
    }
    return reader;
}

Example 15 with FieldReader

use of org.apache.drill.exec.vector.complex.reader.FieldReader in project drill by axbaretto.

the class FunctionConverter method getHolder.

public <T extends DrillFunc> DrillFuncHolder getHolder(AnnotatedClassDescriptor func, ClassLoader classLoader) {
    FunctionTemplate template = func.getAnnotationProxy(FunctionTemplate.class);
    if (template == null) {
        return failure("Class does not declare FunctionTemplate annotation.", func);
    }
    String name = template.name();
    List<String> names = Arrays.asList(template.names());
    if (name.isEmpty() && names.isEmpty()) {
        // none set
        return failure("Must define 'name' or 'names'", func);
    }
    if (!name.isEmpty() && !names.isEmpty()) {
        // both are set
        return failure("Must use only one annotations 'name' or 'names', not both", func);
    }
    // start by getting field information.
    List<ValueReference> params = Lists.newArrayList();
    List<WorkspaceReference> workspaceFields = Lists.newArrayList();
    ValueReference outputField = null;
    for (FieldDescriptor field : func.getFields()) {
        Param param = field.getAnnotationProxy(Param.class);
        Output output = field.getAnnotationProxy(Output.class);
        Workspace workspace = field.getAnnotationProxy(Workspace.class);
        Inject inject = field.getAnnotationProxy(Inject.class);
        Annotation[] annotations = { param, output, workspace, inject };
        int annotationCount = 0;
        for (Annotation annotationDescriptor : annotations) {
            if (annotationDescriptor != null) {
                annotationCount += 1;
            }
        }
        if (annotationCount == 0) {
            return failure("The field must be either a @Param, @Output, @Inject or @Workspace field.", func, field);
        } else if (annotationCount > 1) {
            return failure("The field must be only one of @Param, @Output, @Inject or @Workspace. It currently has more than one of these annotations.", func, field);
        }
        // TODO(Julien): verify there are a few of those and we can load them
        Class<?> fieldClass = field.getFieldClass();
        if (param != null || output != null) {
            // Special processing for @Param FieldReader
            if (param != null && FieldReader.class.isAssignableFrom(fieldClass)) {
                params.add(ValueReference.createFieldReaderRef(field.getName()));
                continue;
            }
            // Special processing for @Output ComplexWriter
            if (output != null && ComplexWriter.class.isAssignableFrom(fieldClass)) {
                if (outputField != null) {
                    return failure("You've declared more than one @Output field.  You must declare one and only @Output field per Function class.", func, field);
                } else {
                    outputField = ValueReference.createComplexWriterRef(field.getName());
                }
                continue;
            }
            // check that param and output are value holders.
            if (!ValueHolder.class.isAssignableFrom(fieldClass)) {
                return failure(String.format("The field doesn't holds value of type %s which does not implement the ValueHolder interface.  All fields of type @Param or @Output must extend this interface..", fieldClass), func, field);
            }
            // get the type field from the value holder.
            MajorType type = null;
            try {
                type = getStaticFieldValue("TYPE", fieldClass, MajorType.class);
            } catch (Exception e) {
                return failure("Failure while trying to access the ValueHolder's TYPE static variable.  All ValueHolders must contain a static TYPE variable that defines their MajorType.", e, func, field);
            }
            ValueReference p = new ValueReference(type, field.getName());
            if (param != null) {
                p.setConstant(param.constant());
                params.add(p);
            } else {
                if (outputField != null) {
                    return failure("You've declared more than one @Output field.  You must declare one and only @Output field per Function class.", func, field);
                } else {
                    outputField = p;
                }
            }
        } else {
            // workspace work.
            boolean isInject = inject != null;
            if (isInject && UdfUtilities.INJECTABLE_GETTER_METHODS.get(fieldClass) == null) {
                return failure(String.format("A %s cannot be injected into a %s," + " available injectable classes are: %s.", fieldClass, DrillFunc.class.getSimpleName(), Joiner.on(",").join(UdfUtilities.INJECTABLE_GETTER_METHODS.keySet())), func, field);
            }
            WorkspaceReference wsReference = new WorkspaceReference(fieldClass, field.getName(), isInject);
            if (!isInject && template.scope() == FunctionScope.POINT_AGGREGATE && !ValueHolder.class.isAssignableFrom(fieldClass)) {
                return failure(String.format("Aggregate function '%s' workspace variable '%s' is of type '%s'. Please change it to Holder type.", func.getClassName(), field.getName(), fieldClass), func, field);
            }
            // If the workspace var is of Holder type, get its MajorType and assign to WorkspaceReference.
            if (ValueHolder.class.isAssignableFrom(fieldClass)) {
                MajorType majorType = null;
                try {
                    majorType = getStaticFieldValue("TYPE", fieldClass, MajorType.class);
                } catch (Exception e) {
                    return failure("Failure while trying to access the ValueHolder's TYPE static variable.  All ValueHolders must contain a static TYPE variable that defines their MajorType.", e, func, field);
                }
                wsReference.setMajorType(majorType);
            }
            workspaceFields.add(wsReference);
        }
    }
    if (outputField == null) {
        return failure("This function declares zero output fields.  A function must declare one output field.", func);
    }
    FunctionInitializer initializer = new FunctionInitializer(func.getClassName(), classLoader);
    try {
        // return holder
        ValueReference[] ps = params.toArray(new ValueReference[params.size()]);
        WorkspaceReference[] works = workspaceFields.toArray(new WorkspaceReference[workspaceFields.size()]);
        FunctionAttributes functionAttributes = new FunctionAttributes(template, ps, outputField, works);
        switch(template.scope()) {
            case POINT_AGGREGATE:
                return new DrillAggFuncHolder(functionAttributes, initializer);
            case SIMPLE:
                return outputField.isComplexWriter() ? new DrillComplexWriterFuncHolder(functionAttributes, initializer) : new DrillSimpleFuncHolder(functionAttributes, initializer);
            case HOLISTIC_AGGREGATE:
            case RANGE_AGGREGATE:
            default:
                return failure("Unsupported Function Type.", func);
        }
    } catch (Exception | NoSuchFieldError | AbstractMethodError ex) {
        return failure("Failure while creating function holder.", ex, func);
    }
}