Examples with ClassTransformationException org.apache.drill.exec.exception.ClassTransformationException used on opensource projects

Example 11 with ClassTransformationException

use of org.apache.drill.exec.exception.ClassTransformationException in project drill by apache.

the class ClassTransformer method getImplementationClass.

public Class<?> getImplementationClass(final QueryClassLoader classLoader, final TemplateClassDefinition<?> templateDefinition, final String entireClass, final String materializedClassName) throws ClassTransformationException {
    // unfortunately, this hasn't been set up at construction time, so we have to do it here
    final ScalarReplacementOption scalarReplacementOption = ScalarReplacementOption.fromString(optionManager.getOption(SCALAR_REPLACEMENT_VALIDATOR));
    try {
        final long t1 = System.nanoTime();
        final ClassSet set = new ClassSet(null, templateDefinition.getTemplateClassName(), materializedClassName);
        final byte[][] implementationClasses = classLoader.getClassByteCode(set.generated, entireClass);
        long totalBytecodeSize = 0;
        Map<String, Pair<byte[], ClassNode>> classesToMerge = Maps.newHashMap();
        for (byte[] clazz : implementationClasses) {
            totalBytecodeSize += clazz.length;
            final ClassNode node = AsmUtil.classFromBytes(clazz, ClassReader.EXPAND_FRAMES);
            if (!AsmUtil.isClassOk(logger, "implementationClasses", node)) {
                throw new IllegalStateException("Problem found with implementationClasses");
            }
            classesToMerge.put(node.name, Pair.of(clazz, node));
        }
        final LinkedList<ClassSet> names = Lists.newLinkedList();
        final Set<ClassSet> namesCompleted = Sets.newHashSet();
        names.add(set);
        while (!names.isEmpty()) {
            final ClassSet nextSet = names.removeFirst();
            if (namesCompleted.contains(nextSet)) {
                continue;
            }
            final ClassNames nextPrecompiled = nextSet.precompiled;
            final byte[] precompiledBytes = byteCodeLoader.getClassByteCodeFromPath(nextPrecompiled.clazz);
            final ClassNames nextGenerated = nextSet.generated;
            // keeps only classes that have not be merged
            Pair<byte[], ClassNode> classNodePair = classesToMerge.remove(nextGenerated.slash);
            final ClassNode generatedNode;
            if (classNodePair != null) {
                generatedNode = classNodePair.getValue();
            } else {
                generatedNode = null;
            }
            /*
         * TODO
         * We're having a problem with some cases of scalar replacement, but we want to get
         * the code in so it doesn't rot anymore.
         *
         *  Here, we use the specified replacement option. The loop will allow us to retry if
         *  we're using TRY.
         */
            MergedClassResult result = null;
            boolean scalarReplace = scalarReplacementOption != ScalarReplacementOption.OFF && entireClass.length() < MAX_SCALAR_REPLACE_CODE_SIZE;
            while (true) {
                try {
                    result = MergeAdapter.getMergedClass(nextSet, precompiledBytes, generatedNode, scalarReplace);
                    break;
                } catch (RuntimeException e) {
                    // if we had a problem without using scalar replacement, then rethrow
                    if (!scalarReplace) {
                        throw e;
                    }
                    // if we did try to use scalar replacement, decide if we need to retry or not
                    if (scalarReplacementOption == ScalarReplacementOption.ON) {
                        // option is forced on, so this is a hard error
                        throw e;
                    }
                    /*
             * We tried to use scalar replacement, with the option to fall back to not using it.
             * Log this failure before trying again without scalar replacement.
             */
                    logger.info("scalar replacement failure (retrying)\n", e);
                    scalarReplace = false;
                }
            }
            for (String s : result.innerClasses) {
                s = s.replace(FileUtils.separatorChar, '.');
                names.add(nextSet.getChild(s));
            }
            classLoader.injectByteCode(nextGenerated.dot, result.bytes);
            namesCompleted.add(nextSet);
        }
        // adds byte code of the classes that have not been merged to make them accessible for outer class
        for (Map.Entry<String, Pair<byte[], ClassNode>> clazz : classesToMerge.entrySet()) {
            classLoader.injectByteCode(clazz.getKey().replace(FileUtils.separatorChar, '.'), clazz.getValue().getKey());
        }
        Class<?> c = classLoader.findClass(set.generated.dot);
        if (templateDefinition.getExternalInterface().isAssignableFrom(c)) {
            logger.debug("Compiled and merged {}: bytecode size = {}, time = {} ms.", c.getSimpleName(), DrillStringUtils.readable(totalBytecodeSize), (System.nanoTime() - t1 + 500_000) / 1_000_000);
            return c;
        }
        throw new ClassTransformationException("The requested class did not implement the expected interface.");
    } catch (CompileException | IOException | ClassNotFoundException e) {
        throw new ClassTransformationException(String.format("Failure generating transformation classes for value: \n %s", entireClass), e);
    }
}

Example 12 with ClassTransformationException

use of org.apache.drill.exec.exception.ClassTransformationException in project drill by apache.

the class CodeCompiler method createInstances.

/**
   * Create multiple instances of the generated class.
   *
   * @param cg code generator for the class to be instantiated.
   * @param count the number of instances desired.
   * @return a list of instances of the generated class.
   * @throws ClassTransformationException general "something is wrong" exception
   * for the Drill compilation chain.
   */
@SuppressWarnings("unchecked")
public <T> List<T> createInstances(final CodeGenerator<?> cg, int count) throws ClassTransformationException {
    if (preferPlainJava && cg.supportsPlainJava()) {
        cg.preferPlainJava(true);
    }
    cg.generate();
    classGenCount++;
    try {
        final GeneratedClassEntry ce;
        if (useCache) {
            ce = cache.get(cg);
            logger.trace(String.format("Class %s found in code cache", cg.getClassName()));
        } else {
            ce = makeClass(cg);
        }
        List<T> tList = Lists.newArrayList();
        for (int i = 0; i < count; i++) {
            tList.add((T) ce.clazz.newInstance());
        }
        return tList;
    } catch (Exception e) {
        throw new ClassTransformationException(e);
    }
}

Example 13 with ClassTransformationException

use of org.apache.drill.exec.exception.ClassTransformationException in project drill by apache.

the class ProjectRecordBatch method setupNewSchema.

@Override
protected boolean setupNewSchema() throws SchemaChangeException {
    if (allocationVectors != null) {
        for (final ValueVector v : allocationVectors) {
            v.clear();
        }
    }
    this.allocationVectors = Lists.newArrayList();
    if (complexWriters != null) {
        container.clear();
    } else {
        container.zeroVectors();
    }
    final List<NamedExpression> exprs = getExpressionList();
    final ErrorCollector collector = new ErrorCollectorImpl();
    final List<TransferPair> transfers = Lists.newArrayList();
    final ClassGenerator<Projector> cg = CodeGenerator.getRoot(Projector.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    cg.getCodeGenerator().plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    cg.getCodeGenerator().saveCodeForDebugging(true);
    final IntHashSet transferFieldIds = new IntHashSet();
    final boolean isAnyWildcard = isAnyWildcard(exprs);
    final ClassifierResult result = new ClassifierResult();
    final boolean classify = isClassificationNeeded(exprs);
    for (int i = 0; i < exprs.size(); i++) {
        final NamedExpression namedExpression = exprs.get(i);
        result.clear();
        if (classify && namedExpression.getExpr() instanceof SchemaPath) {
            classifyExpr(namedExpression, incoming, result);
            if (result.isStar) {
                // The value indicates which wildcard we are processing now
                final Integer value = result.prefixMap.get(result.prefix);
                if (value != null && value.intValue() == 1) {
                    int k = 0;
                    for (final VectorWrapper<?> wrapper : incoming) {
                        final ValueVector vvIn = wrapper.getValueVector();
                        if (k > result.outputNames.size() - 1) {
                            assert false;
                        }
                        // get the renamed column names
                        final String name = result.outputNames.get(k++);
                        if (name == EMPTY_STRING) {
                            continue;
                        }
                        if (isImplicitFileColumn(vvIn)) {
                            continue;
                        }
                        final FieldReference ref = new FieldReference(name);
                        final ValueVector vvOut = container.addOrGet(MaterializedField.create(ref.getAsNamePart().getName(), vvIn.getField().getType()), callBack);
                        final TransferPair tp = vvIn.makeTransferPair(vvOut);
                        transfers.add(tp);
                    }
                } else if (value != null && value.intValue() > 1) {
                    // subsequent wildcards should do a copy of incoming valuevectors
                    int k = 0;
                    for (final VectorWrapper<?> wrapper : incoming) {
                        final ValueVector vvIn = wrapper.getValueVector();
                        final SchemaPath originalPath = SchemaPath.getSimplePath(vvIn.getField().getPath());
                        if (k > result.outputNames.size() - 1) {
                            assert false;
                        }
                        // get the renamed column names
                        final String name = result.outputNames.get(k++);
                        if (name == EMPTY_STRING) {
                            continue;
                        }
                        if (isImplicitFileColumn(vvIn)) {
                            continue;
                        }
                        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(originalPath, incoming, collector, context.getFunctionRegistry());
                        if (collector.hasErrors()) {
                            throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
                        }
                        final MaterializedField outputField = MaterializedField.create(name, expr.getMajorType());
                        final ValueVector vv = container.addOrGet(outputField, callBack);
                        allocationVectors.add(vv);
                        final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
                        final ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
                        final HoldingContainer hc = cg.addExpr(write, ClassGenerator.BlkCreateMode.TRUE_IF_BOUND);
                    }
                }
                continue;
            }
        } else {
            // For the columns which do not needed to be classified,
            // it is still necessary to ensure the output column name is unique
            result.outputNames = Lists.newArrayList();
            final String outputName = getRef(namedExpression).getRootSegment().getPath();
            addToResultMaps(outputName, result, true);
        }
        String outputName = getRef(namedExpression).getRootSegment().getPath();
        if (result != null && result.outputNames != null && result.outputNames.size() > 0) {
            boolean isMatched = false;
            for (int j = 0; j < result.outputNames.size(); j++) {
                if (!result.outputNames.get(j).equals(EMPTY_STRING)) {
                    outputName = result.outputNames.get(j);
                    isMatched = true;
                    break;
                }
            }
            if (!isMatched) {
                continue;
            }
        }
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true, unionTypeEnabled);
        final MaterializedField outputField = MaterializedField.create(outputName, expr.getMajorType());
        if (collector.hasErrors()) {
            throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
        }
        // add value vector to transfer if direct reference and this is allowed, otherwise, add to evaluation stack.
        if (expr instanceof ValueVectorReadExpression && incoming.getSchema().getSelectionVectorMode() == SelectionVectorMode.NONE && !((ValueVectorReadExpression) expr).hasReadPath() && !isAnyWildcard && !transferFieldIds.contains(((ValueVectorReadExpression) expr).getFieldId().getFieldIds()[0])) {
            final ValueVectorReadExpression vectorRead = (ValueVectorReadExpression) expr;
            final TypedFieldId id = vectorRead.getFieldId();
            final ValueVector vvIn = incoming.getValueAccessorById(id.getIntermediateClass(), id.getFieldIds()).getValueVector();
            Preconditions.checkNotNull(incoming);
            final FieldReference ref = getRef(namedExpression);
            final ValueVector vvOut = container.addOrGet(MaterializedField.create(ref.getAsUnescapedPath(), vectorRead.getMajorType()), callBack);
            final TransferPair tp = vvIn.makeTransferPair(vvOut);
            transfers.add(tp);
            transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]);
        } else if (expr instanceof DrillFuncHolderExpr && ((DrillFuncHolderExpr) expr).getHolder().isComplexWriterFuncHolder()) {
            // Lazy initialization of the list of complex writers, if not done yet.
            if (complexWriters == null) {
                complexWriters = Lists.newArrayList();
            } else {
                complexWriters.clear();
            }
            // The reference name will be passed to ComplexWriter, used as the name of the output vector from the writer.
            ((DrillFuncHolderExpr) expr).getFieldReference(namedExpression.getRef());
            cg.addExpr(expr, ClassGenerator.BlkCreateMode.TRUE_IF_BOUND);
            if (complexFieldReferencesList == null) {
                complexFieldReferencesList = Lists.newArrayList();
            }
            // save the field reference for later for getting schema when input is empty
            complexFieldReferencesList.add(namedExpression.getRef());
        } else {
            // need to do evaluation.
            final ValueVector vector = container.addOrGet(outputField, callBack);
            allocationVectors.add(vector);
            final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
            final boolean useSetSafe = !(vector instanceof FixedWidthVector);
            final ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, useSetSafe);
            final HoldingContainer hc = cg.addExpr(write, ClassGenerator.BlkCreateMode.TRUE_IF_BOUND);
            // We cannot do multiple transfers from the same vector. However we still need to instantiate the output vector.
            if (expr instanceof ValueVectorReadExpression) {
                final ValueVectorReadExpression vectorRead = (ValueVectorReadExpression) expr;
                if (!vectorRead.hasReadPath()) {
                    final TypedFieldId id = vectorRead.getFieldId();
                    final ValueVector vvIn = incoming.getValueAccessorById(id.getIntermediateClass(), id.getFieldIds()).getValueVector();
                    vvIn.makeTransferPair(vector);
                }
            }
            logger.debug("Added eval for project expression.");
        }
    }
    try {
        CodeGenerator<Projector> codeGen = cg.getCodeGenerator();
        codeGen.plainJavaCapable(true);
        // Uncomment out this line to debug the generated code.
        //      codeGen.saveCodeForDebugging(true);
        this.projector = context.getImplementationClass(codeGen);
        projector.setup(context, incoming, this, transfers);
    } catch (ClassTransformationException | IOException e) {
        throw new SchemaChangeException("Failure while attempting to load generated class", e);
    }
    if (container.isSchemaChanged()) {
        container.buildSchema(SelectionVectorMode.NONE);
        return true;
    } else {
        return false;
    }
}

Example 14 with ClassTransformationException

use of org.apache.drill.exec.exception.ClassTransformationException in project drill by apache.

the class PartitionSenderRootExec method createClassInstances.

private List<Partitioner> createClassInstances(int actualPartitions) throws SchemaChangeException {
    // set up partitioning function
    final LogicalExpression expr = operator.getExpr();
    final ErrorCollector collector = new ErrorCollectorImpl();
    final ClassGenerator<Partitioner> cg;
    cg = CodeGenerator.getRoot(Partitioner.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    cg.getCodeGenerator().plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    cg.getCodeGenerator().saveCodeForDebugging(true);
    ClassGenerator<Partitioner> cgInner = cg.getInnerGenerator("OutgoingRecordBatch");
    final LogicalExpression materializedExpr = ExpressionTreeMaterializer.materialize(expr, incoming, collector, context.getFunctionRegistry());
    if (collector.hasErrors()) {
        throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
    }
    // generate code to copy from an incoming value vector to the destination partition's outgoing value vector
    JExpression bucket = JExpr.direct("bucket");
    // generate evaluate expression to determine the hash
    ClassGenerator.HoldingContainer exprHolder = cg.addExpr(materializedExpr);
    cg.getEvalBlock().decl(JType.parse(cg.getModel(), "int"), "bucket", exprHolder.getValue().mod(JExpr.lit(outGoingBatchCount)));
    cg.getEvalBlock()._return(cg.getModel().ref(Math.class).staticInvoke("abs").arg(bucket));
    CopyUtil.generateCopies(cgInner, incoming, incoming.getSchema().getSelectionVectorMode() == SelectionVectorMode.FOUR_BYTE);
    try {
        // compile and setup generated code
        List<Partitioner> subPartitioners = context.getImplementationClass(cg, actualPartitions);
        return subPartitioners;
    } catch (ClassTransformationException | IOException e) {
        throw new SchemaChangeException("Failure while attempting to load generated class", e);
    }
}

Example 15 with ClassTransformationException

use of org.apache.drill.exec.exception.ClassTransformationException in project drill by apache.

the class OrderedPartitionRecordBatch method getCopier.

/**
   * Creates a copier that does a project for every Nth record from a VectorContainer incoming into VectorContainer
   * outgoing. Each Ordering in orderings generates a column, and evaluation of the expression associated with each
   * Ordering determines the value of each column. These records will later be sorted based on the values in each
   * column, in the same order as the orderings.
   *
   * @param sv4
   * @param incoming
   * @param outgoing
   * @param orderings
   * @return
   * @throws SchemaChangeException
   */
private SampleCopier getCopier(SelectionVector4 sv4, VectorContainer incoming, VectorContainer outgoing, List<Ordering> orderings, List<ValueVector> localAllocationVectors) throws SchemaChangeException {
    final ErrorCollector collector = new ErrorCollectorImpl();
    final ClassGenerator<SampleCopier> cg = CodeGenerator.getRoot(SampleCopier.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    // Note: disabled for now. This may require some debugging:
    // no tests are available for this operator.
    //    cg.getCodeGenerator().plainOldJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    cg.getCodeGenerator().saveCodeForDebugging(true);
    int i = 0;
    for (Ordering od : orderings) {
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), incoming, collector, context.getFunctionRegistry());
        SchemaPath schemaPath = SchemaPath.getSimplePath("f" + i++);
        TypeProtos.MajorType.Builder builder = TypeProtos.MajorType.newBuilder().mergeFrom(expr.getMajorType()).clearMode().setMode(TypeProtos.DataMode.REQUIRED);
        TypeProtos.MajorType newType = builder.build();
        MaterializedField outputField = MaterializedField.create(schemaPath.getAsUnescapedPath(), newType);
        if (collector.hasErrors()) {
            throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
        }
        @SuppressWarnings("resource") ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
        localAllocationVectors.add(vector);
        TypedFieldId fid = outgoing.add(vector);
        ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
        HoldingContainer hc = cg.addExpr(write);
        cg.getEvalBlock()._if(hc.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
    }
    cg.rotateBlock();
    cg.getEvalBlock()._return(JExpr.TRUE);
    outgoing.buildSchema(BatchSchema.SelectionVectorMode.NONE);
    try {
        SampleCopier sampleCopier = context.getImplementationClass(cg);
        sampleCopier.setupCopier(context, sv4, incoming, outgoing);
        return sampleCopier;
    } catch (ClassTransformationException | IOException e) {
        throw new SchemaChangeException(e);
    }
}