Examples with DrillBuf io.netty.buffer.DrillBuf used on opensource projects

Example 21 with DrillBuf

use of io.netty.buffer.DrillBuf in project drill by axbaretto.

the class WritableBatch method reconstructContainer.

public void reconstructContainer(BufferAllocator allocator, VectorContainer container) {
    Preconditions.checkState(!cleared, "Attempted to reconstruct a container from a WritableBatch after it had been cleared");
    if (buffers.length > 0) {
        /* If we have DrillBuf's associated with value vectors */
        int len = 0;
        for (DrillBuf b : buffers) {
            len += b.capacity();
        }
        @SuppressWarnings("resource") DrillBuf newBuf = allocator.buffer(len);
        try {
            /* Copy data from each buffer into the compound buffer */
            int offset = 0;
            for (DrillBuf buf : buffers) {
                newBuf.setBytes(offset, buf);
                offset += buf.capacity();
                buf.release();
            }
            List<SerializedField> fields = def.getFieldList();
            int bufferOffset = 0;
            /*
         * For each value vector slice up the appropriate size from the compound buffer and load it into the value vector
         */
            int vectorIndex = 0;
            for (VectorWrapper<?> vv : container) {
                SerializedField fmd = fields.get(vectorIndex);
                @SuppressWarnings("resource") ValueVector v = vv.getValueVector();
                @SuppressWarnings("resource") DrillBuf bb = newBuf.slice(bufferOffset, fmd.getBufferLength());
                // v.load(fmd, cbb.slice(bufferOffset, fmd.getBufferLength()));
                v.load(fmd, bb);
                vectorIndex++;
                bufferOffset += fmd.getBufferLength();
            }
        } finally {
            // Any vectors that loaded material from newBuf slices above will retain those.
            newBuf.release(1);
        }
    }
    SelectionVectorMode svMode;
    if (def.hasCarriesTwoByteSelectionVector() && def.getCarriesTwoByteSelectionVector()) {
        svMode = SelectionVectorMode.TWO_BYTE;
    } else {
        svMode = SelectionVectorMode.NONE;
    }
    container.buildSchema(svMode);
    /* Set the record count in the value vector */
    for (VectorWrapper<?> v : container) {
        ValueVector.Mutator m = v.getValueVector().getMutator();
        m.setValueCount(def.getRecordCount());
    }
}

Example 22 with DrillBuf

use of io.netty.buffer.DrillBuf in project drill by axbaretto.

the class QueryResultHandler method batchArrived.

/**
 * Maps internal low-level API protocol to {@link UserResultsListener}-level API protocol.
 * handles query data messages
 */
public void batchArrived(ConnectionThrottle throttle, ByteBuf pBody, ByteBuf dBody) throws RpcException {
    final QueryData queryData = RpcBus.get(pBody, QueryData.PARSER);
    // Current batch coming in.
    final DrillBuf drillBuf = (DrillBuf) dBody;
    final QueryDataBatch batch = new QueryDataBatch(queryData, drillBuf);
    final QueryId queryId = queryData.getQueryId();
    if (logger.isDebugEnabled()) {
        logger.debug("batchArrived: queryId = {}", QueryIdHelper.getQueryId(queryId));
    }
    logger.trace("batchArrived: batch = {}", batch);
    final UserResultsListener resultsListener = newUserResultsListener(queryId);
    // A data case--pass on via dataArrived
    try {
        resultsListener.dataArrived(batch, throttle);
    // That releases batch if successful.
    } catch (Exception e) {
        batch.release();
        resultsListener.submissionFailed(UserException.systemError(e).build(logger));
    }
}

Example 23 with DrillBuf

use of io.netty.buffer.DrillBuf in project drill by axbaretto.

the class RepeatedVarCharOutput method loadRepeatedOffsetAddress.

private void loadRepeatedOffsetAddress() {
    @SuppressWarnings("resource") DrillBuf buf = vector.getOffsetVector().getBuffer();
    checkBuf(buf);
    this.repeatedOffset = buf.memoryAddress() + 4;
    this.repeatedOffsetOriginal = buf.memoryAddress() + 4;
    this.repeatedOffsetMax = buf.memoryAddress() + buf.capacity();
}

Example 24 with DrillBuf

use of io.netty.buffer.DrillBuf in project drill by axbaretto.

the class RepeatedVarCharOutput method loadVarCharDataAddress.

private void loadVarCharDataAddress() {
    @SuppressWarnings("resource") DrillBuf buf = vector.getDataVector().getBuffer();
    checkBuf(buf);
    this.characterData = buf.memoryAddress();
    this.characterDataOriginal = buf.memoryAddress();
    this.characterDataMax = buf.memoryAddress() + buf.capacity();
}

Example 25 with DrillBuf

use of io.netty.buffer.DrillBuf in project drill by axbaretto.

the class TextReader method parseQuotedValue.

/**
 * Recursive function invoked when a quote is encountered. Function also
 * handles the case when there are non-white space characters in the field
 * after the quoted value.
 * @param prev  previous byte read
 * @throws IOException
 */
private void parseQuotedValue(byte prev) throws IOException {
    final byte newLine = this.newLine;
    final byte delimiter = this.delimiter;
    final TextOutput output = this.output;
    final TextInput input = this.input;
    final byte quote = this.quote;
    ch = input.nextCharNoNewLineCheck();
    while (!(prev == quote && (ch == delimiter || ch == newLine || isWhite(ch)))) {
        if (ch != quote) {
            if (prev == quote) {
                // unescaped quote detected
                if (parseUnescapedQuotes) {
                    output.append(quote);
                    output.append(ch);
                    parseQuotedValue(ch);
                    break;
                } else {
                    throw new TextParsingException(context, "Unescaped quote character '" + quote + "' inside quoted value of CSV field. To allow unescaped quotes, set 'parseUnescapedQuotes' to 'true' in the CSV parser settings. Cannot parse CSV input.");
                }
            }
            output.append(ch);
            prev = ch;
        } else if (prev == quoteEscape) {
            output.append(quote);
            prev = NULL_BYTE;
        } else {
            prev = ch;
        }
        ch = input.nextCharNoNewLineCheck();
    }
    // Content after whitespaces may be parsed if 'parseUnescapedQuotes' is enabled.
    if (ch != newLine && ch <= ' ' && ch != delimiter) {
        final DrillBuf workBuf = this.workBuf;
        workBuf.resetWriterIndex();
        do {
            // saves whitespaces after value
            workBuf.writeByte(ch);
            ch = input.nextChar();
            // found a new line, go to next record.
            if (ch == newLine) {
                return;
            }
        } while (ch <= ' ' && ch != delimiter);
        // there's more stuff after the quoted value, not only empty spaces.
        if (!(ch == delimiter || ch == newLine) && parseUnescapedQuotes) {
            output.append(quote);
            for (int i = 0; i < workBuf.writerIndex(); i++) {
                output.append(workBuf.getByte(i));
            }
            // the next character is not the escape character, put it there
            if (ch != quoteEscape) {
                output.append(ch);
            }
            // sets this character as the previous character (may be escaping)
            // calls recursively to keep parsing potentially quoted content
            parseQuotedValue(ch);
        }
    }
    if (!(ch == delimiter || ch == newLine)) {
        throw new TextParsingException(context, "Unexpected character '" + ch + "' following quoted value of CSV field. Expecting '" + delimiter + "'. Cannot parse CSV input.");
    }
}