Examples with IndexStorage org.apache.carbondata.core.datastore.columnar.IndexStorage used on opensource projects

Example 1 with IndexStorage

use of org.apache.carbondata.core.datastore.columnar.IndexStorage in project carbondata by apache.

the class CarbonFactDataHandlerColumnar method processDataRows.

/**
   * generate the NodeHolder from the input rows (one page in case of V3 format)
   */
private NodeHolder processDataRows(List<CarbonRow> dataRows) throws CarbonDataWriterException, KeyGenException {
    if (dataRows.size() == 0) {
        return new NodeHolder();
    }
    TablePage tablePage = new TablePage(model, dataRows.size());
    IndexKey keys = new IndexKey(dataRows.size());
    int rowId = 0;
    // convert row to columnar data
    for (CarbonRow row : dataRows) {
        tablePage.addRow(rowId, row);
        keys.update(rowId, row);
        rowId++;
    }
    // encode and compress dimensions and measure
    // TODO: To make the encoding more transparent to the user, user should be enable to specify
    // the encoding and compression method for each type when creating table.
    Codec codec = new Codec(model.getMeasureDataType());
    IndexStorage[] dimColumns = codec.encodeAndCompressDimensions(tablePage);
    Codec encodedMeasure = codec.encodeAndCompressMeasures(tablePage);
    // prepare nullBitSet for writer, remove this after writer can accept TablePage
    BitSet[] nullBitSet = new BitSet[tablePage.getMeasurePage().length];
    FixLengthColumnPage[] measurePages = tablePage.getMeasurePage();
    for (int i = 0; i < nullBitSet.length; i++) {
        nullBitSet[i] = measurePages[i].getNullBitSet();
    }
    LOGGER.info("Number Of records processed: " + dataRows.size());
    // TODO: writer interface should be modified to use TablePage
    return dataWriter.buildDataNodeHolder(dimColumns, encodedMeasure.getEncodedMeasure(), dataRows.size(), keys.startKey, keys.endKey, encodedMeasure.getCompressionModel(), keys.packedNoDictStartKey, keys.packedNoDictEndKey, nullBitSet);
}