Examples with PathSegment org.apache.drill.common.expression.PathSegment used on opensource projects

Example 11 with PathSegment

use of org.apache.drill.common.expression.PathSegment in project drill by apache.

the class DrillParquetGroupConverter method createFieldConverter.

private Converter createFieldConverter(boolean skipRepeated, Type fieldType, String name, PathSegment colNextChild) {
    Converter converter;
    if (fieldType.isPrimitive()) {
        converter = getConverterForType(name, fieldType.asPrimitiveType());
    } else {
        while (colNextChild != null && !colNextChild.isNamed()) {
            colNextChild = colNextChild.getChild();
        }
        Collection<SchemaPath> columns = colNextChild == null ? Collections.emptyList() : Collections.singletonList(new SchemaPath(colNextChild.getNameSegment()));
        BaseWriter writer;
        GroupType fieldGroupType = fieldType.asGroupType();
        if (ParquetReaderUtility.isLogicalListType(fieldGroupType)) {
            writer = getWriter(name, MapWriter::list, ListWriter::list);
            converter = new DrillParquetGroupConverter(mutator, writer, fieldGroupType, columns, options, containsCorruptedDates, true, converterName);
        } else if (options.getOption(ExecConstants.PARQUET_READER_ENABLE_MAP_SUPPORT_VALIDATOR) && ParquetReaderUtility.isLogicalMapType(fieldGroupType)) {
            writer = getWriter(name, MapWriter::dict, ListWriter::dict);
            converter = new DrillParquetMapGroupConverter(mutator, (DictWriter) writer, fieldGroupType, options, containsCorruptedDates);
        } else if (fieldType.isRepetition(Repetition.REPEATED)) {
            if (skipRepeated) {
                converter = new DrillIntermediateParquetGroupConverter(mutator, baseWriter, fieldGroupType, columns, options, containsCorruptedDates, false, converterName);
            } else {
                writer = getWriter(name, (m, s) -> m.list(s).map(), l -> l.list().map());
                converter = new DrillParquetGroupConverter(mutator, writer, fieldGroupType, columns, options, containsCorruptedDates, false, converterName);
            }
        } else {
            writer = getWriter(name, MapWriter::map, ListWriter::map);
            converter = new DrillParquetGroupConverter(mutator, writer, fieldGroupType, columns, options, containsCorruptedDates, false, converterName);
        }
    }
    return converter;
}

Example 12 with PathSegment

use of org.apache.drill.common.expression.PathSegment in project drill by apache.

the class DrillParquetReader method convertSelectedColumnsToMessageType.

/**
 * Convert SchemaPaths from selectedSchemaPaths and convert to parquet type, and merge into projection schema.
 *
 * @param schema Parquet file schema
 * @param selectedSchemaPaths columns found in schema
 * @return projection schema
 */
private static MessageType convertSelectedColumnsToMessageType(MessageType schema, Set<SchemaPath> selectedSchemaPaths) {
    MessageType projection = null;
    String messageName = schema.getName();
    for (SchemaPath schemaPath : selectedSchemaPaths) {
        List<String> segments = new ArrayList<>();
        PathSegment seg = schemaPath.getRootSegment();
        do {
            segments.add(seg.getNameSegment().getPath());
        } while ((seg = seg.getChild()) != null);
        String[] pathSegments = new String[segments.size()];
        segments.toArray(pathSegments);
        Type t = getSegmentType(pathSegments, 0, schema);
        if (projection == null) {
            projection = new MessageType(messageName, t);
        } else {
            projection = projection.union(new MessageType(messageName, t));
        }
    }
    return projection;
}

Example 13 with PathSegment

use of org.apache.drill.common.expression.PathSegment in project drill by axbaretto.

the class ProjectionSetImpl method addSegment.

private void addSegment(NameSegment rootSegment) {
    String rootKey = rootSegment.getPath().toLowerCase();
    projection.add(rootKey);
    PathSegment child = rootSegment.getChild();
    if (child == null) {
        return;
    }
    if (child.isArray()) {
        // Ignore the [x] array suffix.
        return;
    }
    ProjectionSetImpl map = mapProjections.get(rootKey);
    if (map == null) {
        map = new ProjectionSetImpl();
        mapProjections.put(rootKey, map);
    }
    map.addSegment((NameSegment) child);
}

Example 14 with PathSegment

use of org.apache.drill.common.expression.PathSegment in project drill by axbaretto.

the class BsonRecordReader method ensureAtLeastOneField.

public void ensureAtLeastOneField(ComplexWriter writer) {
    if (!atLeastOneWrite) {
        // if we had no columns, create one empty one so we can return some data
        // for count purposes.
        SchemaPath sp = columns.get(0);
        PathSegment root = sp.getRootSegment();
        BaseWriter.MapWriter fieldWriter = writer.rootAsMap();
        while (root.getChild() != null && !root.getChild().isArray()) {
            fieldWriter = fieldWriter.map(root.getNameSegment().getPath());
            root = root.getChild();
        }
        fieldWriter.integer(root.getNameSegment().getPath());
    }
}

Example 15 with PathSegment

use of org.apache.drill.common.expression.PathSegment in project drill by axbaretto.

the class FieldIdUtil method getFieldIdIfMatches.

public static TypedFieldId getFieldIdIfMatches(ValueVector vector, TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
    if (vector instanceof RepeatedMapVector && seg != null && seg.isArray() && !seg.isLastPath()) {
        if (addToBreadCrumb) {
            addToBreadCrumb = false;
            builder.remainder(seg);
        }
        // skip the first array segment as there is no corresponding child vector.
        seg = seg.getChild();
        // segment.
        if (seg.isArray()) {
            return null;
        }
    }
    if (seg == null) {
        if (addToBreadCrumb) {
            builder.intermediateType(vector.getField().getType());
        }
        return builder.finalType(vector.getField().getType()).build();
    }
    if (seg.isArray()) {
        if (seg.isLastPath()) {
            MajorType type;
            if (vector instanceof AbstractContainerVector) {
                type = ((AbstractContainerVector) vector).getLastPathType();
            } else if (vector instanceof ListVector) {
                type = ((ListVector) vector).getDataVector().getField().getType();
                builder.listVector();
            } else {
                throw new UnsupportedOperationException("FieldIdUtil does not support vector of type " + vector.getField().getType());
            }
            // 
            builder.withIndex().finalType(type);
            // only set remainder when it's the only array segment.
            if (addToBreadCrumb) {
                addToBreadCrumb = false;
                builder.remainder(seg);
            }
            return builder.build();
        } else {
            if (addToBreadCrumb) {
                addToBreadCrumb = false;
                builder.remainder(seg);
            }
        }
    } else {
        if (vector instanceof ListVector) {
            return null;
        }
    }
    ValueVector v;
    if (vector instanceof AbstractContainerVector) {
        VectorWithOrdinal vord = ((AbstractContainerVector) vector).getChildVectorWithOrdinal(seg.isArray() ? null : seg.getNameSegment().getPath());
        if (vord == null) {
            return null;
        }
        v = vord.vector;
        if (addToBreadCrumb) {
            builder.intermediateType(v.getField().getType());
            builder.addId(vord.ordinal);
        }
    } else if (vector instanceof ListVector) {
        v = ((ListVector) vector).getDataVector();
    } else {
        throw new UnsupportedOperationException("FieldIdUtil does not support vector of type " + vector.getField().getType());
    }
    if (v instanceof AbstractContainerVector) {
        // we're looking for a multi path.
        AbstractContainerVector c = (AbstractContainerVector) v;
        return getFieldIdIfMatches(c, builder, addToBreadCrumb, seg.getChild());
    } else if (v instanceof ListVector) {
        ListVector list = (ListVector) v;
        return getFieldIdIfMatches(list, builder, addToBreadCrumb, seg.getChild());
    } else if (v instanceof UnionVector) {
        return getFieldIdIfMatchesUnion((UnionVector) v, builder, addToBreadCrumb, seg.getChild());
    } else {
        if (seg.isNamed()) {
            if (addToBreadCrumb) {
                builder.intermediateType(v.getField().getType());
            }
            builder.finalType(v.getField().getType());
        } else {
            builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build());
        }
        if (seg.isLastPath()) {
            return builder.build();
        } else {
            PathSegment child = seg.getChild();
            if (child.isLastPath() && child.isArray()) {
                if (addToBreadCrumb) {
                    builder.remainder(child);
                }
                builder.withIndex();
                builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build());
                return builder.build();
            } else {
                logger.warn("You tried to request a complex type inside a scalar object or path or type is wrong.");
                return null;
            }
        }
    }
}