Examples with SchemaChangeException org.apache.drill.exec.exception.SchemaChangeException used on opensource projects

Example 46 with SchemaChangeException

use of org.apache.drill.exec.exception.SchemaChangeException in project drill by apache.

the class DrillTestWrapper method addToCombinedVectorResults.

/**
   * Add to result vectors and compare batch schema against expected schema while iterating batches.
   * @param batches
   * @param  expectedSchema: the expected schema the batches should contain. Through SchemaChangeException
   *                       if encounter different batch schema.
   * @return
   * @throws SchemaChangeException
   * @throws UnsupportedEncodingException
   */
public static Map<String, List<Object>> addToCombinedVectorResults(Iterable<VectorAccessible> batches, BatchSchema expectedSchema) throws SchemaChangeException, UnsupportedEncodingException {
    // TODO - this does not handle schema changes
    Map<String, List<Object>> combinedVectors = new TreeMap<>();
    long totalRecords = 0;
    BatchSchema schema = null;
    for (VectorAccessible loader : batches) {
        if (expectedSchema != null) {
            if (!expectedSchema.equals(loader.getSchema())) {
                throw new SchemaChangeException(String.format("Batch schema does not match expected schema\n" + "Actual schema: %s.  Expected schema : %s", loader.getSchema(), expectedSchema));
            }
        }
        // SchemaChangeException, so check/clean throws clause above.
        if (schema == null) {
            schema = loader.getSchema();
            for (MaterializedField mf : schema) {
                combinedVectors.put(SchemaPath.getSimplePath(mf.getPath()).toExpr(), new ArrayList<Object>());
            }
        } else {
            // TODO - actually handle schema changes, this is just to get access to the SelectionVectorMode
            // of the current batch, the check for a null schema is used to only mutate the schema once
            // need to add new vectors and null fill for previous batches? distinction between null and non-existence important?
            schema = loader.getSchema();
        }
        logger.debug("reading batch with " + loader.getRecordCount() + " rows, total read so far " + totalRecords);
        totalRecords += loader.getRecordCount();
        for (VectorWrapper<?> w : loader) {
            String field = SchemaPath.getSimplePath(w.getField().getPath()).toExpr();
            ValueVector[] vectors;
            if (w.isHyper()) {
                vectors = w.getValueVectors();
            } else {
                vectors = new ValueVector[] { w.getValueVector() };
            }
            SelectionVector2 sv2 = null;
            SelectionVector4 sv4 = null;
            switch(schema.getSelectionVectorMode()) {
                case TWO_BYTE:
                    sv2 = loader.getSelectionVector2();
                    break;
                case FOUR_BYTE:
                    sv4 = loader.getSelectionVector4();
                    break;
            }
            if (sv4 != null) {
                for (int j = 0; j < sv4.getCount(); j++) {
                    int complexIndex = sv4.get(j);
                    int batchIndex = complexIndex >> 16;
                    int recordIndexInBatch = complexIndex & 65535;
                    Object obj = vectors[batchIndex].getAccessor().getObject(recordIndexInBatch);
                    if (obj != null) {
                        if (obj instanceof Text) {
                            obj = obj.toString();
                        }
                    }
                    combinedVectors.get(field).add(obj);
                }
            } else {
                for (ValueVector vv : vectors) {
                    for (int j = 0; j < loader.getRecordCount(); j++) {
                        int index;
                        if (sv2 != null) {
                            index = sv2.getIndex(j);
                        } else {
                            index = j;
                        }
                        Object obj = vv.getAccessor().getObject(index);
                        if (obj != null) {
                            if (obj instanceof Text) {
                                obj = obj.toString();
                            }
                        }
                        combinedVectors.get(field).add(obj);
                    }
                }
            }
        }
    }
    return combinedVectors;
}

Example 47 with SchemaChangeException

use of org.apache.drill.exec.exception.SchemaChangeException in project drill by apache.

the class NestedLoopJoinBatch method setupWorker.

/**
   * Method generates the runtime code needed for NLJ. Other than the setup method to set the input and output value
   * vector references we implement three more methods
   * 1. doEval() -> Evaluates if record from left side matches record from the right side
   * 2. emitLeft() -> Project record from the left side
   * 3. emitRight() -> Project record from the right side (which is a hyper container)
   * @return the runtime generated class that implements the NestedLoopJoin interface
   */
private NestedLoopJoin setupWorker() throws IOException, ClassTransformationException, SchemaChangeException {
    final CodeGenerator<NestedLoopJoin> nLJCodeGenerator = CodeGenerator.get(NestedLoopJoin.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    nLJCodeGenerator.plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    nLJCodeGenerator.saveCodeForDebugging(true);
    final ClassGenerator<NestedLoopJoin> nLJClassGenerator = nLJCodeGenerator.getRoot();
    // generate doEval
    final ErrorCollector collector = new ErrorCollectorImpl();
    /*
        Logical expression may contain fields from left and right batches. During code generation (materialization)
        we need to indicate from which input field should be taken.

        Non-equality joins can belong to one of below categories. For example:
        1. Join on non-equality join predicates:
        select * from t1 inner join t2 on (t1.c1 between t2.c1 AND t2.c2) AND (...)
        2. Join with an OR predicate:
        select * from t1 inner join t2 on on t1.c1 = t2.c1 OR t1.c2 = t2.c2
     */
    Map<VectorAccessible, BatchReference> batches = ImmutableMap.<VectorAccessible, BatchReference>builder().put(left, new BatchReference("leftBatch", "leftIndex")).put(rightContainer, new BatchReference("rightContainer", "rightBatchIndex", "rightRecordIndexWithinBatch")).build();
    LogicalExpression materialize = ExpressionTreeMaterializer.materialize(popConfig.getCondition(), batches, collector, context.getFunctionRegistry(), false, false);
    if (collector.hasErrors()) {
        throw new SchemaChangeException(String.format("Failure while trying to materialize join condition. Errors:\n %s.", collector.toErrorString()));
    }
    nLJClassGenerator.addExpr(new ReturnValueExpression(materialize), ClassGenerator.BlkCreateMode.FALSE);
    // generate emitLeft
    nLJClassGenerator.setMappingSet(emitLeftMapping);
    JExpression outIndex = JExpr.direct("outIndex");
    JExpression leftIndex = JExpr.direct("leftIndex");
    int fieldId = 0;
    int outputFieldId = 0;
    // Set the input and output value vector references corresponding to the left batch
    for (MaterializedField field : leftSchema) {
        final TypeProtos.MajorType fieldType = field.getType();
        // Add the vector to the output container
        container.addOrGet(field);
        JVar inVV = nLJClassGenerator.declareVectorValueSetupAndMember("leftBatch", new TypedFieldId(fieldType, false, fieldId));
        JVar outVV = nLJClassGenerator.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(fieldType, false, outputFieldId));
        nLJClassGenerator.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(leftIndex).arg(outIndex).arg(inVV));
        nLJClassGenerator.rotateBlock();
        fieldId++;
        outputFieldId++;
    }
    // generate emitRight
    fieldId = 0;
    nLJClassGenerator.setMappingSet(emitRightMapping);
    JExpression batchIndex = JExpr.direct("batchIndex");
    JExpression recordIndexWithinBatch = JExpr.direct("recordIndexWithinBatch");
    // Set the input and output value vector references corresponding to the right batch
    for (MaterializedField field : rightSchema) {
        final TypeProtos.MajorType inputType = field.getType();
        TypeProtos.MajorType outputType;
        // if join type is LEFT, make sure right batch output fields data mode is optional
        if (popConfig.getJoinType() == JoinRelType.LEFT && inputType.getMode() == TypeProtos.DataMode.REQUIRED) {
            outputType = Types.overrideMode(inputType, TypeProtos.DataMode.OPTIONAL);
        } else {
            outputType = inputType;
        }
        MaterializedField newField = MaterializedField.create(field.getPath(), outputType);
        container.addOrGet(newField);
        JVar inVV = nLJClassGenerator.declareVectorValueSetupAndMember("rightContainer", new TypedFieldId(inputType, true, fieldId));
        JVar outVV = nLJClassGenerator.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType, false, outputFieldId));
        nLJClassGenerator.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(recordIndexWithinBatch).arg(outIndex).arg(inVV.component(batchIndex)));
        nLJClassGenerator.rotateBlock();
        fieldId++;
        outputFieldId++;
    }
    return context.getImplementationClass(nLJCodeGenerator);
}

Example 48 with SchemaChangeException

use of org.apache.drill.exec.exception.SchemaChangeException in project drill by apache.

the class MergingRecordBatch method innerNext.

@Override
public IterOutcome innerNext() {
    if (fragProviders.length == 0) {
        return IterOutcome.NONE;
    }
    boolean schemaChanged = false;
    if (prevBatchWasFull) {
        logger.debug("Outgoing vectors were full on last iteration");
        allocateOutgoing();
        outgoingPosition = 0;
        prevBatchWasFull = false;
    }
    if (!hasMoreIncoming) {
        logger.debug("next() was called after all values have been processed");
        outgoingPosition = 0;
        return IterOutcome.NONE;
    }
    // lazy initialization
    if (!hasRun) {
        // first iteration is always a schema change
        schemaChanged = true;
        // set up each (non-empty) incoming record batch
        final List<RawFragmentBatch> rawBatches = Lists.newArrayList();
        int p = 0;
        for (@SuppressWarnings("unused") final RawFragmentBatchProvider provider : fragProviders) {
            RawFragmentBatch rawBatch;
            // check if there is a batch in temp holder before calling getNext(), as it may have been used when building schema
            if (tempBatchHolder[p] != null) {
                rawBatch = tempBatchHolder[p];
                tempBatchHolder[p] = null;
            } else {
                try {
                    rawBatch = getNext(p);
                } catch (final IOException e) {
                    context.fail(e);
                    return IterOutcome.STOP;
                }
            }
            if (rawBatch == null && !context.shouldContinue()) {
                clearBatches(rawBatches);
                return IterOutcome.STOP;
            }
            assert rawBatch != null : "rawBatch is null although context.shouldContinue() == true";
            if (rawBatch.getHeader().getDef().getRecordCount() != 0) {
                rawBatches.add(rawBatch);
            } else {
                // save an empty batch to use for schema purposes. ignore batch if it contains no fields, and thus no schema
                if (emptyBatch == null && rawBatch.getHeader().getDef().getFieldCount() != 0) {
                    emptyBatch = rawBatch;
                }
                try {
                    while ((rawBatch = getNext(p)) != null && rawBatch.getHeader().getDef().getRecordCount() == 0) {
                    // Do nothing
                    }
                    if (rawBatch == null && !context.shouldContinue()) {
                        clearBatches(rawBatches);
                        return IterOutcome.STOP;
                    }
                } catch (final IOException e) {
                    context.fail(e);
                    clearBatches(rawBatches);
                    return IterOutcome.STOP;
                }
                if (rawBatch != null) {
                    rawBatches.add(rawBatch);
                } else {
                    rawBatches.add(emptyBatch);
                }
            }
            p++;
        }
        // allocate the incoming record batch loaders
        senderCount = rawBatches.size();
        incomingBatches = new RawFragmentBatch[senderCount];
        batchOffsets = new int[senderCount];
        batchLoaders = new RecordBatchLoader[senderCount];
        for (int i = 0; i < senderCount; ++i) {
            incomingBatches[i] = rawBatches.get(i);
            batchLoaders[i] = new RecordBatchLoader(oContext.getAllocator());
        }
        // after this point all batches have moved to incomingBatches
        rawBatches.clear();
        int i = 0;
        for (final RawFragmentBatch batch : incomingBatches) {
            // initialize the incoming batchLoaders
            final UserBitShared.RecordBatchDef rbd = batch.getHeader().getDef();
            try {
                batchLoaders[i].load(rbd, batch.getBody());
            // TODO:  Clean:  DRILL-2933:  That load(...) no longer throws
            // SchemaChangeException, so check/clean catch clause below.
            } catch (final SchemaChangeException e) {
                logger.error("MergingReceiver failed to load record batch from remote host.  {}", e);
                context.fail(e);
                return IterOutcome.STOP;
            }
            batch.release();
            ++batchOffsets[i];
            ++i;
        }
        // Canonicalize each incoming batch, so that vectors are alphabetically sorted based on SchemaPath.
        for (final RecordBatchLoader loader : batchLoaders) {
            loader.canonicalize();
        }
        // Ensure all the incoming batches have the identical schema.
        if (!isSameSchemaAmongBatches(batchLoaders)) {
            context.fail(new SchemaChangeException("Incoming batches for merging receiver have different schemas!"));
            return IterOutcome.STOP;
        }
        // create the outgoing schema and vector container, and allocate the initial batch
        final SchemaBuilder bldr = BatchSchema.newBuilder().setSelectionVectorMode(BatchSchema.SelectionVectorMode.NONE);
        for (final VectorWrapper<?> v : batchLoaders[0]) {
            // add field to the output schema
            bldr.addField(v.getField());
            // allocate a new value vector
            outgoingContainer.addOrGet(v.getField());
        }
        allocateOutgoing();
        outgoingContainer.buildSchema(BatchSchema.SelectionVectorMode.NONE);
        // generate code for merge operations (copy and compare)
        try {
            merger = createMerger();
        } catch (final SchemaChangeException e) {
            logger.error("Failed to generate code for MergingReceiver.  {}", e);
            context.fail(e);
            return IterOutcome.STOP;
        }
        // allocate the priority queue with the generated comparator
        this.pqueue = new PriorityQueue<>(fragProviders.length, new Comparator<Node>() {

            @Override
            public int compare(final Node node1, final Node node2) {
                final int leftIndex = (node1.batchId << 16) + node1.valueIndex;
                final int rightIndex = (node2.batchId << 16) + node2.valueIndex;
                try {
                    return merger.doEval(leftIndex, rightIndex);
                } catch (SchemaChangeException e) {
                    throw new UnsupportedOperationException(e);
                }
            }
        });
        // populate the priority queue with initial values
        for (int b = 0; b < senderCount; ++b) {
            while (batchLoaders[b] != null && batchLoaders[b].getRecordCount() == 0) {
                try {
                    final RawFragmentBatch batch = getNext(b);
                    incomingBatches[b] = batch;
                    if (batch != null) {
                        batchLoaders[b].load(batch.getHeader().getDef(), batch.getBody());
                    } else {
                        batchLoaders[b].clear();
                        batchLoaders[b] = null;
                        if (!context.shouldContinue()) {
                            return IterOutcome.STOP;
                        }
                    }
                } catch (IOException | SchemaChangeException e) {
                    context.fail(e);
                    return IterOutcome.STOP;
                }
            }
            if (batchLoaders[b] != null) {
                pqueue.add(new Node(b, 0));
            }
        }
        hasRun = true;
    // finished lazy initialization
    }
    while (!pqueue.isEmpty()) {
        // pop next value from pq and copy to outgoing batch
        final Node node = pqueue.peek();
        if (!copyRecordToOutgoingBatch(node)) {
            logger.debug("Outgoing vectors space is full; breaking");
            prevBatchWasFull = true;
        }
        pqueue.poll();
        if (node.valueIndex == batchLoaders[node.batchId].getRecordCount() - 1) {
            // reached the end of an incoming record batch
            RawFragmentBatch nextBatch;
            try {
                nextBatch = getNext(node.batchId);
                while (nextBatch != null && nextBatch.getHeader().getDef().getRecordCount() == 0) {
                    nextBatch = getNext(node.batchId);
                }
                assert nextBatch != null || inputCounts[node.batchId] == outputCounts[node.batchId] : String.format("Stream %d input count: %d output count %d", node.batchId, inputCounts[node.batchId], outputCounts[node.batchId]);
                if (nextBatch == null && !context.shouldContinue()) {
                    return IterOutcome.STOP;
                }
            } catch (final IOException e) {
                context.fail(e);
                return IterOutcome.STOP;
            }
            incomingBatches[node.batchId] = nextBatch;
            if (nextBatch == null) {
                // batch is empty
                boolean allBatchesEmpty = true;
                for (final RawFragmentBatch batch : incomingBatches) {
                    // see if all batches are empty so we can return OK_* or NONE
                    if (batch != null) {
                        allBatchesEmpty = false;
                        break;
                    }
                }
                if (allBatchesEmpty) {
                    hasMoreIncoming = false;
                    break;
                }
                // ignored in subsequent iterations.
                if (prevBatchWasFull) {
                    break;
                } else {
                    continue;
                }
            }
            final UserBitShared.RecordBatchDef rbd = incomingBatches[node.batchId].getHeader().getDef();
            try {
                batchLoaders[node.batchId].load(rbd, incomingBatches[node.batchId].getBody());
            // TODO:  Clean:  DRILL-2933:  That load(...) no longer throws
            // SchemaChangeException, so check/clean catch clause below.
            } catch (final SchemaChangeException ex) {
                context.fail(ex);
                return IterOutcome.STOP;
            }
            incomingBatches[node.batchId].release();
            batchOffsets[node.batchId] = 0;
            // add front value from batch[x] to priority queue
            if (batchLoaders[node.batchId].getRecordCount() != 0) {
                pqueue.add(new Node(node.batchId, 0));
            }
        } else {
            pqueue.add(new Node(node.batchId, node.valueIndex + 1));
        }
        if (prevBatchWasFull) {
            break;
        }
    }
    // set the value counts in the outgoing vectors
    for (final VectorWrapper<?> vw : outgoingContainer) {
        vw.getValueVector().getMutator().setValueCount(outgoingPosition);
    }
    if (pqueue.isEmpty()) {
        state = BatchState.DONE;
    }
    if (schemaChanged) {
        return IterOutcome.OK_NEW_SCHEMA;
    } else {
        return IterOutcome.OK;
    }
}

Example 49 with SchemaChangeException

use of org.apache.drill.exec.exception.SchemaChangeException in project drill by apache.

the class HashAggBatch method createAggregatorInternal.

private HashAggregator createAggregatorInternal() throws SchemaChangeException, ClassTransformationException, IOException {
    CodeGenerator<HashAggregator> top = CodeGenerator.get(HashAggregator.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    ClassGenerator<HashAggregator> cg = top.getRoot();
    ClassGenerator<HashAggregator> cgInner = cg.getInnerGenerator("BatchHolder");
    top.plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    top.saveCodeForDebugging(true);
    container.clear();
    int numGroupByExprs = (popConfig.getGroupByExprs() != null) ? popConfig.getGroupByExprs().size() : 0;
    int numAggrExprs = (popConfig.getAggrExprs() != null) ? popConfig.getAggrExprs().size() : 0;
    aggrExprs = new LogicalExpression[numAggrExprs];
    groupByOutFieldIds = new TypedFieldId[numGroupByExprs];
    aggrOutFieldIds = new TypedFieldId[numAggrExprs];
    ErrorCollector collector = new ErrorCollectorImpl();
    int i;
    for (i = 0; i < numGroupByExprs; i++) {
        NamedExpression ne = popConfig.getGroupByExprs().get(i);
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
        if (expr == null) {
            continue;
        }
        final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsNamePart().getName(), expr.getMajorType());
        @SuppressWarnings("resource") ValueVector vv = TypeHelper.getNewVector(outputField, oContext.getAllocator());
        // add this group-by vector to the output container
        groupByOutFieldIds[i] = container.add(vv);
    }
    for (i = 0; i < numAggrExprs; i++) {
        NamedExpression ne = popConfig.getAggrExprs().get(i);
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
        if (expr instanceof IfExpression) {
            throw UserException.unsupportedError(new UnsupportedOperationException("Union type not supported in aggregate functions")).build(logger);
        }
        if (collector.hasErrors()) {
            throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
        }
        if (expr == null) {
            continue;
        }
        final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsNamePart().getName(), expr.getMajorType());
        @SuppressWarnings("resource") ValueVector vv = TypeHelper.getNewVector(outputField, oContext.getAllocator());
        aggrOutFieldIds[i] = container.add(vv);
        aggrExprs[i] = new ValueVectorWriteExpression(aggrOutFieldIds[i], expr, true);
    }
    setupUpdateAggrValues(cgInner);
    setupGetIndex(cg);
    cg.getBlock("resetValues")._return(JExpr.TRUE);
    container.buildSchema(SelectionVectorMode.NONE);
    HashAggregator agg = context.getImplementationClass(top);
    HashTableConfig htConfig = // TODO - fix the validator on this option
    new HashTableConfig((int) context.getOptions().getOption(ExecConstants.MIN_HASH_TABLE_SIZE), HashTable.DEFAULT_LOAD_FACTOR, popConfig.getGroupByExprs(), null, /* no probe exprs */
    comparators);
    agg.setup(popConfig, htConfig, context, this.stats, oContext.getAllocator(), incoming, this, aggrExprs, cgInner.getWorkspaceTypes(), groupByOutFieldIds, this.container);
    return agg;
}

Example 50 with SchemaChangeException

use of org.apache.drill.exec.exception.SchemaChangeException in project drill by apache.

the class TopNBatch method createNewPriorityQueue.

public PriorityQueue createNewPriorityQueue(FragmentContext context, List<Ordering> orderings, VectorAccessible batch, MappingSet mainMapping, MappingSet leftMapping, MappingSet rightMapping) throws ClassTransformationException, IOException, SchemaChangeException {
    CodeGenerator<PriorityQueue> cg = CodeGenerator.get(PriorityQueue.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    cg.plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    cg.saveCodeForDebugging(true);
    ClassGenerator<PriorityQueue> g = cg.getRoot();
    g.setMappingSet(mainMapping);
    for (Ordering od : orderings) {
        // first, we rewrite the evaluation stack for each side of the comparison.
        ErrorCollector collector = new ErrorCollectorImpl();
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), batch, collector, context.getFunctionRegistry(), unionTypeEnabled);
        if (collector.hasErrors()) {
            throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
        }
        g.setMappingSet(leftMapping);
        HoldingContainer left = g.addExpr(expr, ClassGenerator.BlkCreateMode.FALSE);
        g.setMappingSet(rightMapping);
        HoldingContainer right = g.addExpr(expr, ClassGenerator.BlkCreateMode.FALSE);
        g.setMappingSet(mainMapping);
        // next we wrap the two comparison sides and add the expression block for the comparison.
        LogicalExpression fh = FunctionGenerationHelper.getOrderingComparator(od.nullsSortHigh(), left, right, context.getFunctionRegistry());
        HoldingContainer out = g.addExpr(fh, ClassGenerator.BlkCreateMode.FALSE);
        JConditional jc = g.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
        if (od.getDirection() == Direction.ASCENDING) {
            jc._then()._return(out.getValue());
        } else {
            jc._then()._return(out.getValue().minus());
        }
        g.rotateBlock();
    }
    g.rotateBlock();
    g.getEvalBlock()._return(JExpr.lit(0));
    PriorityQueue q = context.getImplementationClass(cg);
    q.init(config.getLimit(), context, oContext.getAllocator(), schema.getSelectionVectorMode() == BatchSchema.SelectionVectorMode.TWO_BYTE);
    return q;
}