Examples with BlockletHeader org.apache.carbondata.format.BlockletHeader used on opensource projects

Example 1 with BlockletHeader

use of org.apache.carbondata.format.BlockletHeader in project carbondata by apache.

the class CarbonStreamRecordReader method nextColumnarBatch.

/**
 * for vector reader, check next columnar batch
 */
private boolean nextColumnarBatch() throws IOException {
    boolean hasNext;
    boolean scanMore = false;
    do {
        // move to the next blocklet
        hasNext = input.nextBlocklet();
        if (hasNext) {
            // read blocklet header
            BlockletHeader header = input.readBlockletHeader();
            if (isScanRequired(header)) {
                scanMore = !scanBlockletAndFillVector(header);
            } else {
                input.skipBlockletData(true);
                scanMore = true;
            }
        } else {
            isFinished = true;
            scanMore = false;
        }
    } while (scanMore);
    return hasNext;
}

Example 2 with BlockletHeader

use of org.apache.carbondata.format.BlockletHeader in project carbondata by apache.

the class CarbonStreamRecordReader method nextRow.

/**
 * check next Row
 */
private boolean nextRow() throws IOException {
    // read row one by one
    try {
        boolean hasNext;
        boolean scanMore = false;
        do {
            hasNext = input.hasNext();
            if (hasNext) {
                if (skipScanData) {
                    input.nextRow();
                    scanMore = false;
                } else {
                    if (useRawRow) {
                        // read raw row for streaming handoff which does not require decode raw row
                        readRawRowFromStream();
                    } else {
                        readRowFromStream();
                    }
                    if (null != filter) {
                        scanMore = !filter.applyFilter(filterRow, carbonTable.getDimensionOrdinalMax());
                    } else {
                        scanMore = false;
                    }
                }
            } else {
                if (input.nextBlocklet()) {
                    BlockletHeader header = input.readBlockletHeader();
                    if (isScanRequired(header)) {
                        if (skipScanData) {
                            input.skipBlockletData(false);
                        } else {
                            input.readBlockletData(header);
                        }
                    } else {
                        input.skipBlockletData(true);
                    }
                    scanMore = true;
                } else {
                    isFinished = true;
                    scanMore = false;
                }
            }
        } while (scanMore);
        return hasNext;
    } catch (FilterUnsupportedException e) {
        throw new IOException("Failed to filter row in detail reader", e);
    }
}

Example 3 with BlockletHeader

use of org.apache.carbondata.format.BlockletHeader in project carbondata by apache.

the class StreamBlockletWriter method apppendBlocklet.

void apppendBlocklet(DataOutputStream outputStream) throws IOException {
    outputStream.write(CarbonStreamOutputFormat.CARBON_SYNC_MARKER);
    BlockletInfo blockletInfo = new BlockletInfo();
    blockletInfo.setNum_rows(getRowIndex() + 1);
    BlockletHeader blockletHeader = new BlockletHeader();
    blockletHeader.setBlocklet_length(getCount());
    blockletHeader.setMutation(MutationType.INSERT);
    blockletHeader.setBlocklet_info(blockletInfo);
    byte[] headerBytes = CarbonUtil.getByteArray(blockletHeader);
    outputStream.writeInt(headerBytes.length);
    outputStream.write(headerBytes);
    byte[] compressed = compressor.compressByte(getBytes(), getCount());
    outputStream.writeInt(compressed.length);
    outputStream.write(compressed);
}

Example 4 with BlockletHeader

use of org.apache.carbondata.format.BlockletHeader in project carbondata by apache.

the class StreamBlockletReader method readBlockletHeader.

BlockletHeader readBlockletHeader() throws IOException {
    int len = readIntFromStream();
    byte[] b = new byte[len];
    if (!readBytesFromStream(b, 0, len)) {
        throw new EOFException("Failed to read blocklet header");
    }
    BlockletHeader header = CarbonUtil.readBlockletHeader(b);
    rowNums = header.getBlocklet_info().getNum_rows();
    rowIndex = 0;
    return header;
}