Examples with SerializedField org.apache.drill.exec.proto.UserBitShared.SerializedField used on opensource projects

Example 16 with SerializedField

use of org.apache.drill.exec.proto.UserBitShared.SerializedField in project drill by axbaretto.

the class RepeatedMapVector method getMetadata.

@Override
public SerializedField getMetadata() {
    SerializedField.Builder builder = getField().getAsBuilder().setBufferLength(getBufferSize()).setValueCount(accessor.getValueCount());
    builder.addChild(offsets.getMetadata());
    for (final ValueVector child : getChildren()) {
        builder.addChild(child.getMetadata());
    }
    return builder.build();
}

Example 17 with SerializedField

use of org.apache.drill.exec.proto.UserBitShared.SerializedField in project drill by apache.

the class AbstractRepeatedMapVector method load.

@Override
public void load(SerializedField metadata, DrillBuf buffer) {
    List<SerializedField> children = metadata.getChildList();
    SerializedField offsetField = children.get(0);
    offsets.load(offsetField, buffer);
    int bufOffset = offsetField.getBufferLength();
    for (int i = 1; i < children.size(); i++) {
        SerializedField child = children.get(i);
        MaterializedField fieldDef = MaterializedField.create(child);
        ValueVector vector = getChild(fieldDef.getName());
        if (vector == null) {
            // if we arrive here, we didn't have a matching vector.
            vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
            putChild(fieldDef.getName(), vector);
        }
        int vectorLength = child.getBufferLength();
        vector.load(child, buffer.slice(bufOffset, vectorLength));
        bufOffset += vectorLength;
    }
    assert bufOffset == buffer.writerIndex();
}

Example 18 with SerializedField

use of org.apache.drill.exec.proto.UserBitShared.SerializedField in project drill by apache.

the class MergingRecordBatch method buildSchema.

@Override
public void buildSchema() {
    // find frag provider that has data to use to build schema, and put in tempBatchHolder for later use
    tempBatchHolder = new RawFragmentBatch[fragProviders.length];
    int i = 0;
    while (true) {
        if (i >= fragProviders.length) {
            state = BatchState.DONE;
            return;
        }
        final RawFragmentBatch batch = getNext(i);
        if (batch == null) {
            checkContinue();
            break;
        }
        if (batch.getHeader().getDef().getFieldCount() == 0) {
            i++;
            continue;
        }
        tempBatchHolder[i] = batch;
        for (final SerializedField field : batch.getHeader().getDef().getFieldList()) {
            final ValueVector v = container.addOrGet(MaterializedField.create(field));
            v.allocateNew();
        }
        break;
    }
    container.buildSchema(SelectionVectorMode.NONE);
    container.setEmpty();
}

Example 19 with SerializedField

use of org.apache.drill.exec.proto.UserBitShared.SerializedField in project drill by apache.

the class RecordBatchLoader method load.

/**
 * Load a record batch from a single buffer.
 *
 * @param def
 *          The definition for the record batch.
 * @param buf
 *          The buffer that holds the data associated with the record batch.
 * @return Whether the schema changed since the previous load.
 */
@SuppressWarnings("resource")
public boolean load(RecordBatchDef def, DrillBuf buf) {
    if (logger.isTraceEnabled()) {
        logger.trace("Loading record batch with def {} and data {}", def, buf);
        logger.trace("Load, ThreadID: {}\n{}", Thread.currentThread().getId(), new StackTrace());
    }
    container.zeroVectors();
    valueCount = def.getRecordCount();
    boolean schemaChanged = schema == null;
    // Load vectors from the batch buffer, while tracking added and/or removed
    // vectors (relative to the previous call) in order to determine whether the
    // the schema has changed since the previous call.
    // Set up to recognize previous fields that no longer exist.
    Map<String, ValueVector> oldFields = CaseInsensitiveMap.newHashMap();
    for (VectorWrapper<?> wrapper : container) {
        ValueVector vector = wrapper.getValueVector();
        oldFields.put(vector.getField().getName(), vector);
    }
    VectorContainer newVectors = new VectorContainer();
    try {
        List<SerializedField> fields = def.getFieldList();
        int bufOffset = 0;
        for (SerializedField field : fields) {
            MaterializedField fieldDef = MaterializedField.create(field);
            ValueVector vector = oldFields.remove(fieldDef.getName());
            if (vector == null) {
                // Field did not exist previously--is schema change.
                schemaChanged = true;
                vector = TypeHelper.getNewVector(fieldDef, allocator);
            } else if (!vector.getField().getType().equals(fieldDef.getType())) {
                // Field had different type before--is schema change.
                // clear previous vector
                vector.clear();
                schemaChanged = true;
                vector = TypeHelper.getNewVector(fieldDef, allocator);
            // If the field is a map or a dict, check if the schema changed.
            } else if ((vector.getField().getType().getMinorType() == MinorType.MAP || vector.getField().getType().getMinorType() == MinorType.DICT) && !isSameSchema(vector.getField().getChildren(), field.getChildList())) {
                // The schema changed. Discard the old one and create a new one.
                schemaChanged = true;
                vector.clear();
                vector = TypeHelper.getNewVector(fieldDef, allocator);
            }
            // Load the vector.
            if (buf == null) {
                // field value alone is sufficient to load the vector
                if (vector instanceof UntypedNullVector) {
                    vector.load(field, null);
                }
            // Schema only
            } else if (field.getValueCount() == 0) {
                AllocationHelper.allocate(vector, 0, 0, 0);
            } else {
                vector.load(field, buf.slice(bufOffset, field.getBufferLength()));
            }
            bufOffset += field.getBufferLength();
            newVectors.add(vector);
        }
        // rebuild the schema.
        SchemaBuilder builder = BatchSchema.newBuilder();
        for (VectorWrapper<?> v : newVectors) {
            builder.addField(v.getField());
        }
        builder.setSelectionVectorMode(BatchSchema.SelectionVectorMode.NONE);
        schema = builder.build();
        newVectors.buildSchema(BatchSchema.SelectionVectorMode.NONE);
        container = newVectors;
        container.setRecordCount(valueCount);
    } catch (final Throwable cause) {
        // We have to clean up new vectors created here and pass over the actual cause.
        // It is upper layer who should adjudicate to call upper layer specific clean up logic.
        VectorAccessibleUtilities.clear(newVectors);
        throw cause;
    } finally {
        if (!oldFields.isEmpty()) {
            schemaChanged = true;
            for (ValueVector vector : oldFields.values()) {
                vector.clear();
            }
        }
    }
    return schemaChanged;
}

Example 20 with SerializedField

use of org.apache.drill.exec.proto.UserBitShared.SerializedField in project drill by apache.

the class VectorAccessibleSerializable method readFromStreamWithContainer.

// Like above, only preserve the original container and list of value-vectors
public void readFromStreamWithContainer(VectorContainer myContainer, InputStream input) throws IOException {
    final VectorContainer container = new VectorContainer();
    final UserBitShared.RecordBatchDef batchDef = UserBitShared.RecordBatchDef.parseDelimitedFrom(input);
    recordCount = batchDef.getRecordCount();
    if (batchDef.hasCarriesTwoByteSelectionVector() && batchDef.getCarriesTwoByteSelectionVector()) {
        if (sv2 == null) {
            sv2 = new SelectionVector2(allocator);
        }
        sv2.allocateNew(recordCount * SelectionVector2.RECORD_SIZE);
        sv2.getBuffer().setBytes(0, input, recordCount * SelectionVector2.RECORD_SIZE);
        svMode = BatchSchema.SelectionVectorMode.TWO_BYTE;
    }
    final List<ValueVector> vectorList = Lists.newArrayList();
    final List<SerializedField> fieldList = batchDef.getFieldList();
    for (SerializedField metaData : fieldList) {
        final int dataLength = metaData.getBufferLength();
        final MaterializedField field = MaterializedField.create(metaData);
        final DrillBuf buf = allocator.buffer(dataLength);
        final ValueVector vector;
        try {
            buf.writeBytes(input, dataLength);
            vector = TypeHelper.getNewVector(field, allocator);
            vector.load(metaData, buf);
        } finally {
            buf.release();
        }
        vectorList.add(vector);
    }
    container.addCollection(vectorList);
    container.setRecordCount(recordCount);
    // transfer the vectors
    myContainer.transferIn(container);
    myContainer.buildSchema(svMode);
    myContainer.setRecordCount(recordCount);
    /*
    // for debugging -- show values from the first row
    Object tmp0 = (myContainer).getValueAccessorById(NullableVarCharVector.class, 0).getValueVector();
    Object tmp1 = (myContainer).getValueAccessorById(NullableVarCharVector.class, 1).getValueVector();
    Object tmp2 = (myContainer).getValueAccessorById(NullableBigIntVector.class, 2).getValueVector();
    if (tmp0 != null && tmp1 != null && tmp2 != null) {
      NullableVarCharVector vv0 = ((NullableVarCharVector) tmp0);
      NullableVarCharVector vv1 = ((NullableVarCharVector) tmp1);
      NullableBigIntVector vv2 = ((NullableBigIntVector) tmp2);

      try {
        logger.info("HASH AGG: Got a row = {} , {} , {}", vv0.getAccessor().get(0), vv1.getAccessor().get(0), vv2.getAccessor().get(0));
      } catch (Exception e) { logger.info("HASH AGG: Got an exception = {}",e); }
    }
    else { logger.info("HASH AGG: got nulls !!!"); }
    */
    va = myContainer;
}