Examples with SelectionVectorMode org.apache.drill.exec.record.BatchSchema.SelectionVectorMode used on opensource projects

Example 1 with SelectionVectorMode

use of org.apache.drill.exec.record.BatchSchema.SelectionVectorMode in project drill by apache.

the class VectorValidator method validate.

public static void validate(RecordBatch batch) {
    int count = batch.getRecordCount();
    long hash = 12345;
    SelectionVectorMode mode = batch.getSchema().getSelectionVectorMode();
    switch(mode) {
        case NONE:
            {
                for (VectorWrapper w : batch) {
                    ValueVector v = w.getValueVector();
                    for (int i = 0; i < count; i++) {
                        Object obj = v.getAccessor().getObject(i);
                        if (obj != null) {
                            hash = obj.hashCode() ^ hash;
                        }
                    }
                }
                break;
            }
        case TWO_BYTE:
            {
                for (VectorWrapper w : batch) {
                    ValueVector v = w.getValueVector();
                    for (int i = 0; i < count; i++) {
                        int index = batch.getSelectionVector2().getIndex(i);
                        Object obj = v.getAccessor().getObject(index);
                        if (obj != null) {
                            hash = obj.hashCode() ^ hash;
                        }
                    }
                }
                break;
            }
        case FOUR_BYTE:
            {
                for (VectorWrapper w : batch) {
                    ValueVector[] vv = w.getValueVectors();
                    for (int i = 0; i < count; i++) {
                        int index = batch.getSelectionVector4().get(i);
                        ValueVector v = vv[index >> 16];
                        Object obj = v.getAccessor().getObject(index & 65535);
                        if (obj != null) {
                            hash = obj.hashCode() ^ hash;
                        }
                    }
                }
            }
    }
    if (hash == 0) {
    //      System.out.println(hash);
    }
}

Example 2 with SelectionVectorMode

use of org.apache.drill.exec.record.BatchSchema.SelectionVectorMode in project drill by apache.

the class WritableBatch method reconstructContainer.

public void reconstructContainer(BufferAllocator allocator, VectorContainer container) {
    Preconditions.checkState(!cleared, "Attempted to reconstruct a container from a WritableBatch after it had been cleared");
    if (buffers.length > 0) {
        /* If we have DrillBuf's associated with value vectors */
        int len = 0;
        for (DrillBuf b : buffers) {
            len += b.capacity();
        }
        DrillBuf newBuf = allocator.buffer(len);
        try {
            /* Copy data from each buffer into the compound buffer */
            int offset = 0;
            for (DrillBuf buf : buffers) {
                newBuf.setBytes(offset, buf);
                offset += buf.capacity();
                buf.release();
            }
            List<SerializedField> fields = def.getFieldList();
            int bufferOffset = 0;
            /*
         * For each value vector slice up the appropriate size from the compound buffer and load it into the value vector
         */
            int vectorIndex = 0;
            for (VectorWrapper<?> vv : container) {
                SerializedField fmd = fields.get(vectorIndex);
                ValueVector v = vv.getValueVector();
                DrillBuf bb = newBuf.slice(bufferOffset, fmd.getBufferLength());
                //        v.load(fmd, cbb.slice(bufferOffset, fmd.getBufferLength()));
                v.load(fmd, bb);
                vectorIndex++;
                bufferOffset += fmd.getBufferLength();
            }
        } finally {
            // Any vectors that loaded material from newBuf slices above will retain those.
            newBuf.release(1);
        }
    }
    SelectionVectorMode svMode;
    if (def.hasCarriesTwoByteSelectionVector() && def.getCarriesTwoByteSelectionVector()) {
        svMode = SelectionVectorMode.TWO_BYTE;
    } else {
        svMode = SelectionVectorMode.NONE;
    }
    container.buildSchema(svMode);
    /* Set the record count in the value vector */
    for (VectorWrapper<?> v : container) {
        ValueVector.Mutator m = v.getValueVector().getMutator();
        m.setValueCount(def.getRecordCount());
    }
}

Example 3 with SelectionVectorMode

use of org.apache.drill.exec.record.BatchSchema.SelectionVectorMode in project drill by apache.

the class PartitionerTemplate method setup.

@Override
public final void setup(FragmentContext context, RecordBatch incoming, HashPartitionSender popConfig, OperatorStats stats, OperatorContext oContext, int start, int end) throws SchemaChangeException {
    this.incoming = incoming;
    this.stats = stats;
    this.start = start;
    this.end = end;
    doSetup(context, incoming, null);
    // allocated.
    if (popConfig.getDestinations().size() > 1000) {
        // Always keep the recordCount as (2^x) - 1 to better utilize the memory allocation in ValueVectors
        outgoingRecordBatchSize = (DEFAULT_RECORD_BATCH_SIZE + 1) / 2 - 1;
    }
    int fieldId = 0;
    for (MinorFragmentEndpoint destination : popConfig.getDestinations()) {
        // create outgoingBatches only for subset of Destination Points
        if (fieldId >= start && fieldId < end) {
            logger.debug("start: {}, count: {}, fieldId: {}", start, end, fieldId);
            outgoingBatches.add(newOutgoingRecordBatch(stats, popConfig, context.getDataTunnel(destination.getEndpoint()), context, oContext.getAllocator(), destination.getId()));
        }
        fieldId++;
    }
    for (OutgoingRecordBatch outgoingRecordBatch : outgoingBatches) {
        outgoingRecordBatch.initializeBatch();
    }
    SelectionVectorMode svMode = incoming.getSchema().getSelectionVectorMode();
    switch(svMode) {
        case FOUR_BYTE:
            this.sv4 = incoming.getSelectionVector4();
            break;
        case TWO_BYTE:
            this.sv2 = incoming.getSelectionVector2();
            break;
        case NONE:
            break;
        default:
            throw new UnsupportedOperationException("Unknown selection vector mode: " + svMode.toString());
    }
}

Example 4 with SelectionVectorMode

use of org.apache.drill.exec.record.BatchSchema.SelectionVectorMode in project drill by apache.

the class SelectionVectorPrelVisitor method visitPrel.

@Override
public Prel visitPrel(Prel prel, Void value) throws RuntimeException {
    SelectionVectorMode[] encodings = prel.getSupportedEncodings();
    List<RelNode> children = Lists.newArrayList();
    for (Prel child : prel) {
        child = child.accept(this, null);
        children.add(convert(encodings, child));
    }
    return (Prel) prel.copy(prel.getTraitSet(), children);
}

Example 5 with SelectionVectorMode

use of org.apache.drill.exec.record.BatchSchema.SelectionVectorMode in project drill by apache.

the class PartitionerTemplate method partitionBatch.

@Override
public void partitionBatch(RecordBatch incoming) throws IOException {
    SelectionVectorMode svMode = incoming.getSchema().getSelectionVectorMode();
    // Keeping the for loop inside the case to avoid case evaluation for each record.
    switch(svMode) {
        case NONE:
            for (int recordId = 0; recordId < incoming.getRecordCount(); ++recordId) {
                doCopy(recordId);
            }
            break;
        case TWO_BYTE:
            for (int recordId = 0; recordId < incoming.getRecordCount(); ++recordId) {
                int svIndex = sv2.getIndex(recordId);
                doCopy(svIndex);
            }
            break;
        case FOUR_BYTE:
            for (int recordId = 0; recordId < incoming.getRecordCount(); ++recordId) {
                int svIndex = sv4.get(recordId);
                doCopy(svIndex);
            }
            break;
        default:
            throw new UnsupportedOperationException("Unknown selection vector mode: " + svMode.toString());
    }
}