use of org.apache.carbondata.core.datastore.page.FixLengthColumnPage in project carbondata by apache.
the class CarbonFactDataHandlerColumnar method processDataRows.
/**
* generate the NodeHolder from the input rows (one page in case of V3 format)
*/
private NodeHolder processDataRows(List<CarbonRow> dataRows) throws CarbonDataWriterException, KeyGenException {
if (dataRows.size() == 0) {
return new NodeHolder();
}
TablePage tablePage = new TablePage(model, dataRows.size());
IndexKey keys = new IndexKey(dataRows.size());
int rowId = 0;
// convert row to columnar data
for (CarbonRow row : dataRows) {
tablePage.addRow(rowId, row);
keys.update(rowId, row);
rowId++;
}
// encode and compress dimensions and measure
// TODO: To make the encoding more transparent to the user, user should be enable to specify
// the encoding and compression method for each type when creating table.
Codec codec = new Codec(model.getMeasureDataType());
IndexStorage[] dimColumns = codec.encodeAndCompressDimensions(tablePage);
Codec encodedMeasure = codec.encodeAndCompressMeasures(tablePage);
// prepare nullBitSet for writer, remove this after writer can accept TablePage
BitSet[] nullBitSet = new BitSet[tablePage.getMeasurePage().length];
FixLengthColumnPage[] measurePages = tablePage.getMeasurePage();
for (int i = 0; i < nullBitSet.length; i++) {
nullBitSet[i] = measurePages[i].getNullBitSet();
}
LOGGER.info("Number Of records processed: " + dataRows.size());
// TODO: writer interface should be modified to use TablePage
return dataWriter.buildDataNodeHolder(dimColumns, encodedMeasure.getEncodedMeasure(), dataRows.size(), keys.startKey, keys.endKey, encodedMeasure.getCompressionModel(), keys.packedNoDictStartKey, keys.packedNoDictEndKey, nullBitSet);
}
Aggregations