Examples with MaterializedField org.apache.drill.exec.record.MaterializedField used on opensource projects

Example 1 with MaterializedField

use of org.apache.drill.exec.record.MaterializedField in project drill by apache.

the class VectorAccessibleSerializable method readVectors.

@SuppressWarnings("resource")
private void readVectors(InputStream input, RecordBatchDef batchDef) throws IOException {
    final VectorContainer container = new VectorContainer();
    final List<ValueVector> vectorList = Lists.newArrayList();
    final List<SerializedField> fieldList = batchDef.getFieldList();
    for (SerializedField metaData : fieldList) {
        final int dataLength = metaData.getBufferLength();
        final MaterializedField field = MaterializedField.create(metaData);
        final DrillBuf buf = allocator.read(dataLength, input);
        final ValueVector vector = TypeHelper.getNewVector(field, allocator);
        vector.load(metaData, buf);
        // Vector now owns the buffer
        buf.release();
        vectorList.add(vector);
    }
    container.addCollection(vectorList);
    container.buildSchema(svMode);
    container.setRecordCount(recordCount);
    va = container;
}

Example 2 with MaterializedField

use of org.apache.drill.exec.record.MaterializedField in project drill by apache.

the class DumpCat method showSingleBatch.

private void showSingleBatch(VectorAccessibleSerializable vcSerializable, boolean showHeader) {
    final VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
    /* show the header of the batch */
    if (showHeader) {
        System.out.println(getBatchMetaInfo(vcSerializable).toString());
        System.out.println("Schema Information");
        for (final VectorWrapper w : vectorContainer) {
            final MaterializedField field = w.getValueVector().getField();
            System.out.println(String.format("name : %s, minor_type : %s, data_mode : %s", field.getPath(), field.getType().getMinorType().toString(), field.isNullable() ? "nullable" : "non-nullable"));
        }
    }
    /* show the contents in the batch */
    VectorUtil.showVectorAccessibleContent(vectorContainer);
}

Example 3 with MaterializedField

use of org.apache.drill.exec.record.MaterializedField in project drill by apache.

the class TestDrillbitResilience method assertDrillbitsOk.

/**
   * Check that all the drillbits are ok.
   * <p/>
   * <p>The current implementation does this by counting the number of drillbits using a query.
   */
private static void assertDrillbitsOk() {
    final SingleRowListener listener = new SingleRowListener() {

        private final BufferAllocator bufferAllocator = RootAllocatorFactory.newRoot(zkHelper.getConfig());

        private final RecordBatchLoader loader = new RecordBatchLoader(bufferAllocator);

        @Override
        public void rowArrived(final QueryDataBatch queryResultBatch) {
            // load the single record
            final QueryData queryData = queryResultBatch.getHeader();
            try {
                loader.load(queryData.getDef(), queryResultBatch.getData());
            // TODO:  Clean:  DRILL-2933:  That load(...) no longer throws
            // SchemaChangeException, so check/clean catch clause below.
            } catch (final SchemaChangeException e) {
                fail(e.toString());
            }
            assertEquals(1, loader.getRecordCount());
            // there should only be one column
            final BatchSchema batchSchema = loader.getSchema();
            assertEquals(1, batchSchema.getFieldCount());
            // the column should be an integer
            final MaterializedField countField = batchSchema.getColumn(0);
            final MinorType fieldType = countField.getType().getMinorType();
            assertEquals(MinorType.BIGINT, fieldType);
            // get the column value
            final VectorWrapper<?> vw = loader.iterator().next();
            final Object obj = vw.getValueVector().getAccessor().getObject(0);
            assertTrue(obj instanceof Long);
            final Long countValue = (Long) obj;
            // assume this means all the drillbits are still ok
            assertEquals(drillbits.size(), countValue.intValue());
            loader.clear();
        }

        @Override
        public void cleanup() {
            DrillAutoCloseables.closeNoChecked(bufferAllocator);
        }
    };
    try {
        QueryTestUtil.testWithListener(drillClient, QueryType.SQL, "select count(*) from sys.memory", listener);
        listener.waitForCompletion();
        final QueryState state = listener.getQueryState();
        assertTrue(String.format("QueryState should be COMPLETED (and not %s).", state), state == QueryState.COMPLETED);
    } catch (final Exception e) {
        throw new RuntimeException("Couldn't query active drillbits", e);
    }
    final List<DrillPBError> errorList = listener.getErrorList();
    assertTrue("There should not be any errors when checking if Drillbits are OK.", errorList.isEmpty());
}

Example 4 with MaterializedField

use of org.apache.drill.exec.record.MaterializedField in project drill by apache.

the class HiveAbstractReader method setup.

@Override
public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException {
    // initializes "reader"
    final Callable<Void> readerInitializer = new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            init();
            return null;
        }
    };
    final ListenableFuture<Void> result = context.runCallableAs(proxyUgi, readerInitializer);
    try {
        result.get();
    } catch (InterruptedException e) {
        result.cancel(true);
        // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the
        // interruption and respond to it if it wants to.
        Thread.currentThread().interrupt();
    } catch (ExecutionException e) {
        throw ExecutionSetupException.fromThrowable(e.getMessage(), e);
    }
    try {
        final OptionManager options = fragmentContext.getOptions();
        for (int i = 0; i < selectedColumnNames.size(); i++) {
            MajorType type = HiveUtilities.getMajorTypeFromHiveTypeInfo(selectedColumnTypes.get(i), options);
            MaterializedField field = MaterializedField.create(selectedColumnNames.get(i), type);
            Class<? extends ValueVector> vvClass = TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode());
            vectors.add(output.addField(field, vvClass));
        }
        for (int i = 0; i < selectedPartitionNames.size(); i++) {
            MajorType type = HiveUtilities.getMajorTypeFromHiveTypeInfo(selectedPartitionTypes.get(i), options);
            MaterializedField field = MaterializedField.create(selectedPartitionNames.get(i), type);
            Class<? extends ValueVector> vvClass = TypeHelper.getValueVectorClass(field.getType().getMinorType(), field.getDataMode());
            pVectors.add(output.addField(field, vvClass));
        }
    } catch (SchemaChangeException e) {
        throw new ExecutionSetupException(e);
    }
}

Example 5 with MaterializedField

use of org.apache.drill.exec.record.MaterializedField in project drill by apache.

the class ChainedHashTable method createAndSetupHashTable.

public HashTable createAndSetupHashTable(TypedFieldId[] outKeyFieldIds) throws ClassTransformationException, IOException, SchemaChangeException {
    CodeGenerator<HashTable> top = CodeGenerator.get(HashTable.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    top.plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    // This code is called from generated code, so to step into this code,
    // persist the code generated in HashAggBatch also.
    //  top.saveCodeForDebugging(true);
    ClassGenerator<HashTable> cg = top.getRoot();
    ClassGenerator<HashTable> cgInner = cg.getInnerGenerator("BatchHolder");
    LogicalExpression[] keyExprsBuild = new LogicalExpression[htConfig.getKeyExprsBuild().size()];
    LogicalExpression[] keyExprsProbe = null;
    boolean isProbe = (htConfig.getKeyExprsProbe() != null);
    if (isProbe) {
        keyExprsProbe = new LogicalExpression[htConfig.getKeyExprsProbe().size()];
    }
    ErrorCollector collector = new ErrorCollectorImpl();
    // original ht container from which others may be cloned
    VectorContainer htContainerOrig = new VectorContainer();
    LogicalExpression[] htKeyExprs = new LogicalExpression[htConfig.getKeyExprsBuild().size()];
    TypedFieldId[] htKeyFieldIds = new TypedFieldId[htConfig.getKeyExprsBuild().size()];
    int i = 0;
    for (NamedExpression ne : htConfig.getKeyExprsBuild()) {
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incomingBuild, collector, context.getFunctionRegistry());
        if (collector.hasErrors()) {
            throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
        }
        if (expr == null) {
            continue;
        }
        keyExprsBuild[i] = expr;
        i++;
    }
    if (isProbe) {
        i = 0;
        for (NamedExpression ne : htConfig.getKeyExprsProbe()) {
            final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incomingProbe, collector, context.getFunctionRegistry());
            if (collector.hasErrors()) {
                throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
            }
            if (expr == null) {
                continue;
            }
            keyExprsProbe[i] = expr;
            i++;
        }
        JoinUtils.addLeastRestrictiveCasts(keyExprsProbe, incomingProbe, keyExprsBuild, incomingBuild, context);
    }
    i = 0;
    /*
     * Once the implicit casts have been added, create the value vectors for the corresponding
     * type and add it to the hash table's container.
     * Note: Adding implicit casts may have a minor impact on the memory foot print. For example
     * if we have a join condition with bigint on the probe side and int on the build side then
     * after this change we will be allocating a bigint vector in the hashtable instead of an int
     * vector.
     */
    for (NamedExpression ne : htConfig.getKeyExprsBuild()) {
        LogicalExpression expr = keyExprsBuild[i];
        final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsUnescapedPath(), expr.getMajorType());
        @SuppressWarnings("resource") ValueVector vv = TypeHelper.getNewVector(outputField, allocator);
        htKeyFieldIds[i] = htContainerOrig.add(vv);
        i++;
    }
    // generate code for isKeyMatch(), setValue(), getHash() and outputRecordKeys()
    setupIsKeyMatchInternal(cgInner, KeyMatchIncomingBuildMapping, KeyMatchHtableMapping, keyExprsBuild, htConfig.getComparators(), htKeyFieldIds);
    setupIsKeyMatchInternal(cgInner, KeyMatchIncomingProbeMapping, KeyMatchHtableProbeMapping, keyExprsProbe, htConfig.getComparators(), htKeyFieldIds);
    setupSetValue(cgInner, keyExprsBuild, htKeyFieldIds);
    if (outgoing != null) {
        if (outKeyFieldIds.length > htConfig.getKeyExprsBuild().size()) {
            throw new IllegalArgumentException("Mismatched number of output key fields.");
        }
    }
    setupOutputRecordKeys(cgInner, htKeyFieldIds, outKeyFieldIds);
    setupGetHash(cg, /* use top level code generator for getHash */
    GetHashIncomingBuildMapping, incomingBuild, keyExprsBuild, false);
    setupGetHash(cg, /* use top level code generator for getHash */
    GetHashIncomingProbeMapping, incomingProbe, keyExprsProbe, true);
    HashTable ht = context.getImplementationClass(top);
    ht.setup(htConfig, context, allocator, incomingBuild, incomingProbe, outgoing, htContainerOrig);
    return ht;
}