Examples with SchemaChangeException org.apache.drill.exec.exception.SchemaChangeException used on opensource projects

Example 6 with SchemaChangeException

use of org.apache.drill.exec.exception.SchemaChangeException in project drill by apache.

the class FilterRecordBatch method generateSV4Filterer.

protected Filterer generateSV4Filterer() throws SchemaChangeException {
    final ErrorCollector collector = new ErrorCollectorImpl();
    final List<TransferPair> transfers = Lists.newArrayList();
    final ClassGenerator<Filterer> cg = CodeGenerator.getRoot(Filterer.TEMPLATE_DEFINITION4, context.getFunctionRegistry(), context.getOptions());
    final LogicalExpression expr = ExpressionTreeMaterializer.materialize(popConfig.getExpr(), incoming, collector, context.getFunctionRegistry());
    if (collector.hasErrors()) {
        throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
    }
    cg.addExpr(new ReturnValueExpression(expr), ClassGenerator.BlkCreateMode.FALSE);
    for (final VectorWrapper<?> vw : incoming) {
        for (final ValueVector vv : vw.getValueVectors()) {
            final TransferPair pair = vv.getTransferPair(oContext.getAllocator());
            container.add(pair.getTo());
            transfers.add(pair);
        }
    }
    // allocate outgoing sv4
    container.buildSchema(SelectionVectorMode.FOUR_BYTE);
    try {
        final TransferPair[] tx = transfers.toArray(new TransferPair[transfers.size()]);
        final Filterer filter = context.getImplementationClass(cg);
        filter.setup(context, incoming, this, tx);
        return filter;
    } catch (ClassTransformationException | IOException e) {
        throw new SchemaChangeException("Failure while attempting to load generated class", e);
    }
}

Example 7 with SchemaChangeException

use of org.apache.drill.exec.exception.SchemaChangeException in project drill by apache.

the class FilterRecordBatch method generateSV2Filterer.

protected Filterer generateSV2Filterer() throws SchemaChangeException {
    final ErrorCollector collector = new ErrorCollectorImpl();
    final List<TransferPair> transfers = Lists.newArrayList();
    final ClassGenerator<Filterer> cg = CodeGenerator.getRoot(Filterer.TEMPLATE_DEFINITION2, context.getFunctionRegistry(), context.getOptions());
    final LogicalExpression expr = ExpressionTreeMaterializer.materialize(popConfig.getExpr(), incoming, collector, context.getFunctionRegistry(), false, unionTypeEnabled);
    if (collector.hasErrors()) {
        throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
    }
    cg.addExpr(new ReturnValueExpression(expr), ClassGenerator.BlkCreateMode.FALSE);
    for (final VectorWrapper<?> v : incoming) {
        final TransferPair pair = v.getValueVector().makeTransferPair(container.addOrGet(v.getField(), callBack));
        transfers.add(pair);
    }
    try {
        final TransferPair[] tx = transfers.toArray(new TransferPair[transfers.size()]);
        CodeGenerator<Filterer> codeGen = cg.getCodeGenerator();
        codeGen.plainJavaCapable(true);
        // Uncomment out this line to debug the generated code.
        //    cg.saveCodeForDebugging(true);
        final Filterer filter = context.getImplementationClass(codeGen);
        filter.setup(context, incoming, this, tx);
        return filter;
    } catch (ClassTransformationException | IOException e) {
        throw new SchemaChangeException("Failure while attempting to load generated class", e);
    }
}

Example 8 with SchemaChangeException

use of org.apache.drill.exec.exception.SchemaChangeException in project drill by apache.

the class UnionAllRecordBatch method doWork.

@SuppressWarnings("resource")
private IterOutcome doWork() throws ClassTransformationException, IOException, SchemaChangeException {
    if (allocationVectors != null) {
        for (ValueVector v : allocationVectors) {
            v.clear();
        }
    }
    allocationVectors = Lists.newArrayList();
    transfers.clear();
    // If both sides of Union-All are empty
    if (unionAllInput.isBothSideEmpty()) {
        for (int i = 0; i < outputFields.size(); ++i) {
            final String colName = outputFields.get(i).getPath();
            final MajorType majorType = MajorType.newBuilder().setMinorType(MinorType.INT).setMode(DataMode.OPTIONAL).build();
            MaterializedField outputField = MaterializedField.create(colName, majorType);
            ValueVector vv = container.addOrGet(outputField, callBack);
            allocationVectors.add(vv);
        }
        container.buildSchema(BatchSchema.SelectionVectorMode.NONE);
        return IterOutcome.OK_NEW_SCHEMA;
    }
    final ClassGenerator<UnionAller> cg = CodeGenerator.getRoot(UnionAller.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    cg.getCodeGenerator().plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    cg.getCodeGenerator().saveCodeForDebugging(true);
    int index = 0;
    for (VectorWrapper<?> vw : current) {
        ValueVector vvIn = vw.getValueVector();
        // get the original input column names
        SchemaPath inputPath = SchemaPath.getSimplePath(vvIn.getField().getPath());
        // get the renamed column names
        SchemaPath outputPath = SchemaPath.getSimplePath(outputFields.get(index).getPath());
        final ErrorCollector collector = new ErrorCollectorImpl();
        // cast data types (Minortype or DataMode)
        if (hasSameTypeAndMode(outputFields.get(index), vw.getValueVector().getField())) {
            // Transfer column
            MajorType outputFieldType = outputFields.get(index).getType();
            MaterializedField outputField = MaterializedField.create(outputPath.getAsUnescapedPath(), outputFieldType);
            /*
          todo: Fix if condition when DRILL-4824 is merged
          If condition should be changed to:
          `if (outputFields.get(index).getPath().equals(inputPath.getAsUnescapedPath())) {`
          DRILL-5419 has changed condition to correct one but this caused regression (DRILL-5521).
          Root cause is missing indication of child column in map types when it is null.
          DRILL-4824 is re-working json reader implementation, including map types and will fix this problem.
          Reverting condition to previous one to avoid regression till DRILL-4824 is merged.
          Unit test - TestJsonReader.testKvgenWithUnionAll().
         */
            if (outputFields.get(index).getPath().equals(inputPath)) {
                ValueVector vvOut = container.addOrGet(outputField);
                TransferPair tp = vvIn.makeTransferPair(vvOut);
                transfers.add(tp);
            // Copy data in order to rename the column
            } else {
                final LogicalExpression expr = ExpressionTreeMaterializer.materialize(inputPath, current, collector, context.getFunctionRegistry());
                if (collector.hasErrors()) {
                    throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
                }
                ValueVector vv = container.addOrGet(outputField, callBack);
                allocationVectors.add(vv);
                TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
                ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
                cg.addExpr(write);
            }
        // Cast is necessary
        } else {
            LogicalExpression expr = ExpressionTreeMaterializer.materialize(inputPath, current, collector, context.getFunctionRegistry());
            if (collector.hasErrors()) {
                throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
            }
            // cast to the one with the least restriction
            if (vvIn.getField().getType().getMode() == DataMode.REQUIRED && outputFields.get(index).getType().getMode() != DataMode.REQUIRED) {
                expr = ExpressionTreeMaterializer.convertToNullableType(expr, vvIn.getField().getType().getMinorType(), context.getFunctionRegistry(), collector);
                if (collector.hasErrors()) {
                    throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
                }
            }
            // Insert a cast before the Union operation
            if (vvIn.getField().getType().getMinorType() != outputFields.get(index).getType().getMinorType()) {
                expr = ExpressionTreeMaterializer.addCastExpression(expr, outputFields.get(index).getType(), context.getFunctionRegistry(), collector);
                if (collector.hasErrors()) {
                    throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
                }
            }
            final MaterializedField outputField = MaterializedField.create(outputPath.getAsUnescapedPath(), expr.getMajorType());
            ValueVector vector = container.addOrGet(outputField, callBack);
            allocationVectors.add(vector);
            TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
            boolean useSetSafe = !(vector instanceof FixedWidthVector);
            ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, useSetSafe);
            cg.addExpr(write);
        }
        ++index;
    }
    unionall = context.getImplementationClass(cg.getCodeGenerator());
    unionall.setup(context, current, this, transfers);
    if (!schemaAvailable) {
        container.buildSchema(BatchSchema.SelectionVectorMode.NONE);
        schemaAvailable = true;
    }
    if (!doAlloc()) {
        return IterOutcome.OUT_OF_MEMORY;
    }
    recordCount = unionall.unionRecords(0, current.getRecordCount(), 0);
    setValueCount(recordCount);
    return IterOutcome.OK;
}

Example 9 with SchemaChangeException

use of org.apache.drill.exec.exception.SchemaChangeException in project drill by apache.

the class UnorderedReceiverBatch method next.

@Override
public IterOutcome next() {
    batchLoader.resetRecordCount();
    stats.startProcessing();
    try {
        RawFragmentBatch batch;
        try {
            stats.startWait();
            batch = getNextBatch();
            // skip over empty batches. we do this since these are basically control messages.
            while (batch != null && batch.getHeader().getDef().getRecordCount() == 0 && (!first || batch.getHeader().getDef().getFieldCount() == 0)) {
                batch = getNextBatch();
            }
        } finally {
            stats.stopWait();
        }
        first = false;
        if (batch == null) {
            batchLoader.clear();
            if (!context.shouldContinue()) {
                return IterOutcome.STOP;
            }
            return IterOutcome.NONE;
        }
        if (context.isOverMemoryLimit()) {
            return IterOutcome.OUT_OF_MEMORY;
        }
        //      logger.debug("Next received batch {}", batch);
        final RecordBatchDef rbd = batch.getHeader().getDef();
        final boolean schemaChanged = batchLoader.load(rbd, batch.getBody());
        // TODO:  Clean:  DRILL-2933:  That load(...) no longer throws
        // SchemaChangeException, so check/clean catch clause below.
        stats.addLongStat(Metric.BYTES_RECEIVED, batch.getByteCount());
        batch.release();
        if (schemaChanged) {
            this.schema = batchLoader.getSchema();
            stats.batchReceived(0, rbd.getRecordCount(), true);
            return IterOutcome.OK_NEW_SCHEMA;
        } else {
            stats.batchReceived(0, rbd.getRecordCount(), false);
            return IterOutcome.OK;
        }
    } catch (SchemaChangeException | IOException ex) {
        context.fail(ex);
        return IterOutcome.STOP;
    } finally {
        stats.stopProcessing();
    }
}

Example 10 with SchemaChangeException

use of org.apache.drill.exec.exception.SchemaChangeException in project drill by apache.

the class ExternalSortBatch method processBatch.

/**
   * Process the converted incoming batch by adding it to the in-memory store
   * of data, or spilling data to disk when necessary.
   */
@SuppressWarnings("resource")
private void processBatch() {
    if (incoming.getRecordCount() == 0) {
        return;
    }
    // Determine actual sizes of the incoming batch before taking
    // ownership. Allows us to figure out if we need to spill first,
    // to avoid overflowing memory simply due to ownership transfer.
    RecordBatchSizer sizer = analyzeIncomingBatch();
    if (isSpillNeeded(sizer.actualSize())) {
        spillFromMemory();
    }
    // Sanity check. We should now be below the buffer memory maximum.
    long startMem = allocator.getAllocatedMemory();
    if (startMem > bufferMemoryPool) {
        logger.error("ERROR: Failed to spill above buffer limit. Buffer pool = {}, memory = {}", bufferMemoryPool, startMem);
    }
    // Convert the incoming batch to the agreed-upon schema.
    // No converted batch means we got an empty input batch.
    // Converting the batch transfers memory ownership to our
    // allocator. This gives a round-about way to learn the batch
    // size: check the before and after memory levels, then use
    // the difference as the batch size, in bytes.
    VectorContainer convertedBatch = convertBatch();
    if (convertedBatch == null) {
        return;
    }
    SelectionVector2 sv2;
    try {
        sv2 = makeSelectionVector();
    } catch (Exception e) {
        convertedBatch.clear();
        throw e;
    }
    // Compute batch size, including allocation of an sv2.
    long endMem = allocator.getAllocatedMemory();
    long batchSize = endMem - startMem;
    int count = sv2.getCount();
    inputRecordCount += count;
    inputBatchCount++;
    totalInputBytes += sizer.actualSize();
    if (minimumBufferSpace == 0) {
        minimumBufferSpace = endMem;
    } else {
        minimumBufferSpace = Math.min(minimumBufferSpace, endMem);
    }
    stats.setLongStat(Metric.MIN_BUFFER, minimumBufferSpace);
    // Update the size based on the actual record count, not
    // the effective count as given by the selection vector
    // (which may exclude some records due to filtering.)
    updateMemoryEstimates(batchSize, sizer);
    // Sort the incoming batch using either the original selection vector,
    // or a new one created here.
    SingleBatchSorter sorter;
    sorter = opCodeGen.getSorter(convertedBatch);
    try {
        sorter.setup(context, sv2, convertedBatch);
    } catch (SchemaChangeException e) {
        convertedBatch.clear();
        throw UserException.unsupportedError(e).message("Unexpected schema change.").build(logger);
    }
    try {
        sorter.sort(sv2);
    } catch (SchemaChangeException e) {
        convertedBatch.clear();
        throw UserException.unsupportedError(e).message("Unexpected schema change.").build(logger);
    }
    RecordBatchData rbd = new RecordBatchData(convertedBatch, allocator);
    try {
        rbd.setSv2(sv2);
        bufferedBatches.add(new BatchGroup.InputBatch(rbd.getContainer(), rbd.getSv2(), oContext, sizer.netSize()));
        if (peakNumBatches < bufferedBatches.size()) {
            peakNumBatches = bufferedBatches.size();
            stats.setLongStat(Metric.PEAK_BATCHES_IN_MEMORY, peakNumBatches);
        }
    } catch (Throwable t) {
        rbd.clear();
        throw t;
    }
}