Examples with TypedFieldId org.apache.drill.exec.record.TypedFieldId used on opensource projects

Example 36 with TypedFieldId

use of org.apache.drill.exec.record.TypedFieldId in project drill by axbaretto.

the class FieldIdUtil method getFieldId.

public static TypedFieldId getFieldId(ValueVector vector, int id, SchemaPath expectedPath, boolean hyper) {
    if (!expectedPath.getRootSegment().getPath().equalsIgnoreCase(vector.getField().getName())) {
        return null;
    }
    PathSegment seg = expectedPath.getRootSegment();
    TypedFieldId.Builder builder = TypedFieldId.newBuilder();
    if (hyper) {
        builder.hyper();
    }
    if (vector instanceof UnionVector) {
        builder.addId(id).remainder(expectedPath.getRootSegment().getChild());
        List<MinorType> minorTypes = ((UnionVector) vector).getSubTypes();
        MajorType.Builder majorTypeBuilder = MajorType.newBuilder().setMinorType(MinorType.UNION);
        for (MinorType type : minorTypes) {
            majorTypeBuilder.addSubType(type);
        }
        MajorType majorType = majorTypeBuilder.build();
        builder.intermediateType(majorType);
        if (seg.isLastPath()) {
            builder.finalType(majorType);
            return builder.build();
        } else {
            return getFieldIdIfMatchesUnion((UnionVector) vector, builder, false, seg.getChild());
        }
    } else if (vector instanceof ListVector) {
        ListVector list = (ListVector) vector;
        builder.intermediateType(vector.getField().getType());
        builder.addId(id);
        return getFieldIdIfMatches(list, builder, true, expectedPath.getRootSegment().getChild());
    } else if (vector instanceof AbstractContainerVector) {
        // we're looking for a multi path.
        AbstractContainerVector c = (AbstractContainerVector) vector;
        builder.intermediateType(vector.getField().getType());
        builder.addId(id);
        return getFieldIdIfMatches(c, builder, true, expectedPath.getRootSegment().getChild());
    } else {
        builder.intermediateType(vector.getField().getType());
        builder.addId(id);
        builder.finalType(vector.getField().getType());
        if (seg.isLastPath()) {
            return builder.build();
        } else {
            PathSegment child = seg.getChild();
            if (child.isArray() && child.isLastPath()) {
                builder.remainder(child);
                builder.withIndex();
                builder.finalType(vector.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build());
                return builder.build();
            } else {
                return null;
            }
        }
    }
}

Example 37 with TypedFieldId

use of org.apache.drill.exec.record.TypedFieldId in project drill by axbaretto.

the class TestOperatorRecordBatch method testBatchAccessor.

/**
 * The record batch abstraction has a bunch of methods to work with a vector container.
 * Rather than simply exposing the container itself, the batch instead exposes various
 * container operations. Probably an artifact of its history. In any event, make
 * sure those methods are passed through to the container accessor.
 */
@Test
public void testBatchAccessor() {
    BatchSchema schema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).build();
    SingleRowSet rs = fixture.rowSetBuilder(schema).addRow(10, "fred").addRow(20, "wilma").build();
    MockOperatorExec opExec = new MockOperatorExec(rs.container());
    opExec.nextCalls = 1;
    try (OperatorRecordBatch opBatch = makeOpBatch(opExec)) {
        assertEquals(IterOutcome.OK_NEW_SCHEMA, opBatch.next());
        assertEquals(schema, opBatch.getSchema());
        assertEquals(2, opBatch.getRecordCount());
        assertSame(rs.container(), opBatch.getOutgoingContainer());
        Iterator<VectorWrapper<?>> iter = opBatch.iterator();
        assertEquals("a", iter.next().getValueVector().getField().getName());
        assertEquals("b", iter.next().getValueVector().getField().getName());
        // Not a full test of the schema path; just make sure that the
        // pass-through to the Vector Container works.
        SchemaPath path = SchemaPath.create(NamePart.newBuilder().setName("a").build());
        TypedFieldId id = opBatch.getValueVectorId(path);
        assertEquals(MinorType.INT, id.getFinalType().getMinorType());
        assertEquals(1, id.getFieldIds().length);
        assertEquals(0, id.getFieldIds()[0]);
        path = SchemaPath.create(NamePart.newBuilder().setName("b").build());
        id = opBatch.getValueVectorId(path);
        assertEquals(MinorType.VARCHAR, id.getFinalType().getMinorType());
        assertEquals(1, id.getFieldIds().length);
        assertEquals(1, id.getFieldIds()[0]);
        // Sanity check of getValueAccessorById()
        VectorWrapper<?> w = opBatch.getValueAccessorById(IntVector.class, 0);
        assertNotNull(w);
        assertEquals("a", w.getValueVector().getField().getName());
        w = opBatch.getValueAccessorById(VarCharVector.class, 1);
        assertNotNull(w);
        assertEquals("b", w.getValueVector().getField().getName());
        try {
            opBatch.getSelectionVector2();
            fail();
        } catch (UnsupportedOperationException e) {
        // Expected
        }
        try {
            opBatch.getSelectionVector4();
            fail();
        } catch (UnsupportedOperationException e) {
        // Expected
        }
    } catch (Exception e) {
        fail(e.getMessage());
    }
    assertTrue(opExec.closeCalled);
}

Example 38 with TypedFieldId

use of org.apache.drill.exec.record.TypedFieldId in project drill by apache.

the class ProjectRecordBatch method setupNewSchema.

@Override
protected boolean setupNewSchema() throws SchemaChangeException {
    if (allocationVectors != null) {
        for (final ValueVector v : allocationVectors) {
            v.clear();
        }
    }
    this.allocationVectors = Lists.newArrayList();
    if (complexWriters != null) {
        container.clear();
    } else {
        container.zeroVectors();
    }
    final List<NamedExpression> exprs = getExpressionList();
    final ErrorCollector collector = new ErrorCollectorImpl();
    final List<TransferPair> transfers = Lists.newArrayList();
    final ClassGenerator<Projector> cg = CodeGenerator.getRoot(Projector.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    cg.getCodeGenerator().plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    cg.getCodeGenerator().saveCodeForDebugging(true);
    final IntHashSet transferFieldIds = new IntHashSet();
    final boolean isAnyWildcard = isAnyWildcard(exprs);
    final ClassifierResult result = new ClassifierResult();
    final boolean classify = isClassificationNeeded(exprs);
    for (int i = 0; i < exprs.size(); i++) {
        final NamedExpression namedExpression = exprs.get(i);
        result.clear();
        if (classify && namedExpression.getExpr() instanceof SchemaPath) {
            classifyExpr(namedExpression, incoming, result);
            if (result.isStar) {
                // The value indicates which wildcard we are processing now
                final Integer value = result.prefixMap.get(result.prefix);
                if (value != null && value.intValue() == 1) {
                    int k = 0;
                    for (final VectorWrapper<?> wrapper : incoming) {
                        final ValueVector vvIn = wrapper.getValueVector();
                        if (k > result.outputNames.size() - 1) {
                            assert false;
                        }
                        // get the renamed column names
                        final String name = result.outputNames.get(k++);
                        if (name == EMPTY_STRING) {
                            continue;
                        }
                        if (isImplicitFileColumn(vvIn)) {
                            continue;
                        }
                        final FieldReference ref = new FieldReference(name);
                        final ValueVector vvOut = container.addOrGet(MaterializedField.create(ref.getAsNamePart().getName(), vvIn.getField().getType()), callBack);
                        final TransferPair tp = vvIn.makeTransferPair(vvOut);
                        transfers.add(tp);
                    }
                } else if (value != null && value.intValue() > 1) {
                    // subsequent wildcards should do a copy of incoming valuevectors
                    int k = 0;
                    for (final VectorWrapper<?> wrapper : incoming) {
                        final ValueVector vvIn = wrapper.getValueVector();
                        final SchemaPath originalPath = SchemaPath.getSimplePath(vvIn.getField().getPath());
                        if (k > result.outputNames.size() - 1) {
                            assert false;
                        }
                        // get the renamed column names
                        final String name = result.outputNames.get(k++);
                        if (name == EMPTY_STRING) {
                            continue;
                        }
                        if (isImplicitFileColumn(vvIn)) {
                            continue;
                        }
                        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(originalPath, incoming, collector, context.getFunctionRegistry());
                        if (collector.hasErrors()) {
                            throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
                        }
                        final MaterializedField outputField = MaterializedField.create(name, expr.getMajorType());
                        final ValueVector vv = container.addOrGet(outputField, callBack);
                        allocationVectors.add(vv);
                        final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
                        final ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
                        final HoldingContainer hc = cg.addExpr(write, ClassGenerator.BlkCreateMode.TRUE_IF_BOUND);
                    }
                }
                continue;
            }
        } else {
            // For the columns which do not needed to be classified,
            // it is still necessary to ensure the output column name is unique
            result.outputNames = Lists.newArrayList();
            final String outputName = getRef(namedExpression).getRootSegment().getPath();
            addToResultMaps(outputName, result, true);
        }
        String outputName = getRef(namedExpression).getRootSegment().getPath();
        if (result != null && result.outputNames != null && result.outputNames.size() > 0) {
            boolean isMatched = false;
            for (int j = 0; j < result.outputNames.size(); j++) {
                if (!result.outputNames.get(j).equals(EMPTY_STRING)) {
                    outputName = result.outputNames.get(j);
                    isMatched = true;
                    break;
                }
            }
            if (!isMatched) {
                continue;
            }
        }
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true, unionTypeEnabled);
        final MaterializedField outputField = MaterializedField.create(outputName, expr.getMajorType());
        if (collector.hasErrors()) {
            throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
        }
        // add value vector to transfer if direct reference and this is allowed, otherwise, add to evaluation stack.
        if (expr instanceof ValueVectorReadExpression && incoming.getSchema().getSelectionVectorMode() == SelectionVectorMode.NONE && !((ValueVectorReadExpression) expr).hasReadPath() && !isAnyWildcard && !transferFieldIds.contains(((ValueVectorReadExpression) expr).getFieldId().getFieldIds()[0])) {
            final ValueVectorReadExpression vectorRead = (ValueVectorReadExpression) expr;
            final TypedFieldId id = vectorRead.getFieldId();
            final ValueVector vvIn = incoming.getValueAccessorById(id.getIntermediateClass(), id.getFieldIds()).getValueVector();
            Preconditions.checkNotNull(incoming);
            final FieldReference ref = getRef(namedExpression);
            final ValueVector vvOut = container.addOrGet(MaterializedField.create(ref.getAsUnescapedPath(), vectorRead.getMajorType()), callBack);
            final TransferPair tp = vvIn.makeTransferPair(vvOut);
            transfers.add(tp);
            transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]);
        } else if (expr instanceof DrillFuncHolderExpr && ((DrillFuncHolderExpr) expr).getHolder().isComplexWriterFuncHolder()) {
            // Lazy initialization of the list of complex writers, if not done yet.
            if (complexWriters == null) {
                complexWriters = Lists.newArrayList();
            } else {
                complexWriters.clear();
            }
            // The reference name will be passed to ComplexWriter, used as the name of the output vector from the writer.
            ((DrillFuncHolderExpr) expr).getFieldReference(namedExpression.getRef());
            cg.addExpr(expr, ClassGenerator.BlkCreateMode.TRUE_IF_BOUND);
            if (complexFieldReferencesList == null) {
                complexFieldReferencesList = Lists.newArrayList();
            }
            // save the field reference for later for getting schema when input is empty
            complexFieldReferencesList.add(namedExpression.getRef());
        } else {
            // need to do evaluation.
            final ValueVector vector = container.addOrGet(outputField, callBack);
            allocationVectors.add(vector);
            final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
            final boolean useSetSafe = !(vector instanceof FixedWidthVector);
            final ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, useSetSafe);
            final HoldingContainer hc = cg.addExpr(write, ClassGenerator.BlkCreateMode.TRUE_IF_BOUND);
            // We cannot do multiple transfers from the same vector. However we still need to instantiate the output vector.
            if (expr instanceof ValueVectorReadExpression) {
                final ValueVectorReadExpression vectorRead = (ValueVectorReadExpression) expr;
                if (!vectorRead.hasReadPath()) {
                    final TypedFieldId id = vectorRead.getFieldId();
                    final ValueVector vvIn = incoming.getValueAccessorById(id.getIntermediateClass(), id.getFieldIds()).getValueVector();
                    vvIn.makeTransferPair(vector);
                }
            }
            logger.debug("Added eval for project expression.");
        }
    }
    try {
        CodeGenerator<Projector> codeGen = cg.getCodeGenerator();
        codeGen.plainJavaCapable(true);
        // Uncomment out this line to debug the generated code.
        //      codeGen.saveCodeForDebugging(true);
        this.projector = context.getImplementationClass(codeGen);
        projector.setup(context, incoming, this, transfers);
    } catch (ClassTransformationException | IOException e) {
        throw new SchemaChangeException("Failure while attempting to load generated class", e);
    }
    if (container.isSchemaChanged()) {
        container.buildSchema(SelectionVectorMode.NONE);
        return true;
    } else {
        return false;
    }
}

Example 39 with TypedFieldId

use of org.apache.drill.exec.record.TypedFieldId in project drill by apache.

the class HashAggTemplate method setup.

@Override
public void setup(HashAggregate hashAggrConfig, HashTableConfig htConfig, FragmentContext context, OperatorStats stats, BufferAllocator allocator, RecordBatch incoming, HashAggBatch outgoing, LogicalExpression[] valueExprs, List<TypedFieldId> valueFieldIds, TypedFieldId[] groupByOutFieldIds, VectorContainer outContainer) throws SchemaChangeException, ClassTransformationException, IOException {
    if (valueExprs == null || valueFieldIds == null) {
        throw new IllegalArgumentException("Invalid aggr value exprs or workspace variables.");
    }
    if (valueFieldIds.size() < valueExprs.length) {
        throw new IllegalArgumentException("Wrong number of workspace variables.");
    }
    //    this.context = context;
    this.stats = stats;
    this.allocator = allocator;
    this.incoming = incoming;
    //    this.schema = incoming.getSchema();
    this.outgoing = outgoing;
    this.outContainer = outContainer;
    // TODO:  This functionality will be added later.
    if (hashAggrConfig.getGroupByExprs().size() == 0) {
        throw new IllegalArgumentException("Currently, hash aggregation is only applicable if there are group-by " + "expressions.");
    }
    this.htIdxHolder = new IndexPointer();
    this.outStartIdxHolder = new IndexPointer();
    this.outNumRecordsHolder = new IndexPointer();
    materializedValueFields = new MaterializedField[valueFieldIds.size()];
    if (valueFieldIds.size() > 0) {
        int i = 0;
        FieldReference ref = new FieldReference("dummy", ExpressionPosition.UNKNOWN, valueFieldIds.get(0).getIntermediateType());
        for (TypedFieldId id : valueFieldIds) {
            materializedValueFields[i++] = MaterializedField.create(ref.getAsNamePart().getName(), id.getIntermediateType());
        }
    }
    ChainedHashTable ht = new ChainedHashTable(htConfig, context, allocator, incoming, null, /* no incoming probe */
    outgoing);
    this.htable = ht.createAndSetupHashTable(groupByOutFieldIds);
    numGroupByOutFields = groupByOutFieldIds.length;
    batchHolders = new ArrayList<BatchHolder>();
    // First BatchHolder is created when the first put request is received.
    doSetup(incoming);
}

Example 40 with TypedFieldId

use of org.apache.drill.exec.record.TypedFieldId in project drill by apache.

the class ExpressionTest method testSpecial.

@Test
public void testSpecial(@Injectable final RecordBatch batch, @Injectable ValueVector vector) throws Exception {
    final TypeProtos.MajorType type = Types.optional(MinorType.INT);
    final TypedFieldId tfid = new TypedFieldId(type, false, 0);
    new NonStrictExpectations() {

        @NonStrict
        VectorWrapper<?> wrapper;

        {
            batch.getValueVectorId(new SchemaPath("alpha", ExpressionPosition.UNKNOWN));
            result = tfid;
            batch.getValueAccessorById(IntVector.class, tfid.getFieldIds());
            result = wrapper;
            wrapper.getValueVector();
            result = new IntVector(MaterializedField.create("result", type), RootAllocatorFactory.newRoot(c));
        }
    };
    System.out.println(getExpressionCode("1 + 1", batch));
}