Examples with NodeHolder org.apache.carbondata.core.util.NodeHolder used on opensource projects

Example 1 with NodeHolder

use of org.apache.carbondata.core.util.NodeHolder in project carbondata by apache.

the class CarbonFactDataWriterImplV3 method writeDataToFile.

/**
   * Below method will be used to write data in carbon data file
   * Data Format
   * <Column1 Data ChunkV3><Column1<Page1><Page2><Page3><Page4>>
   * <Column2 Data ChunkV3><Column2<Page1><Page2><Page3><Page4>>
   * <Column3 Data ChunkV3><Column3<Page1><Page2><Page3><Page4>>
   * <Column4 Data ChunkV3><Column4<Page1><Page2><Page3><Page4>>
   * Each page will contain column data, Inverted index and rle index
   *
   * @param channel
   * @param dataChunkBytes
   */
private void writeDataToFile(FileChannel channel, byte[][] dataChunkBytes) {
    long offset = 0;
    // write the header
    try {
        if (fileChannel.size() == 0) {
            // below code is to write the file header
            byte[] fileHeader = CarbonUtil.getByteArray(CarbonMetadataUtil.getFileHeader(true, thriftColumnSchemaList, dataWriterVo.getSchemaUpdatedTimeStamp()));
            ByteBuffer buffer = ByteBuffer.allocate(fileHeader.length);
            buffer.put(fileHeader);
            buffer.flip();
            fileChannel.write(buffer);
        }
        offset = channel.size();
    } catch (IOException e) {
        throw new CarbonDataWriterException("Problem while getting the file channel size");
    }
    // to maintain the offset of each data chunk in blocklet
    List<Long> currentDataChunksOffset = new ArrayList<>();
    // to maintain the length of each data chunk in blocklet
    List<Integer> currentDataChunksLength = new ArrayList<>();
    // get the node holder list
    List<NodeHolder> nodeHolderList = dataWriterHolder.getNodeHolder();
    int numberOfDimension = nodeHolderList.get(0).getKeyArray().length;
    int numberOfMeasures = nodeHolderList.get(0).getDataArray().length;
    NodeHolder nodeHolder = null;
    ByteBuffer buffer = null;
    int bufferSize = 0;
    long dimensionOffset = 0;
    long measureOffset = 0;
    int numberOfRows = 0;
    // calculate the number of rows in each blocklet
    for (int j = 0; j < nodeHolderList.size(); j++) {
        numberOfRows += nodeHolderList.get(j).getEntryCount();
    }
    try {
        for (int i = 0; i < numberOfDimension; i++) {
            currentDataChunksOffset.add(offset);
            currentDataChunksLength.add(dataChunkBytes[i].length);
            buffer = ByteBuffer.allocate(dataChunkBytes[i].length);
            buffer.put(dataChunkBytes[i]);
            buffer.flip();
            fileChannel.write(buffer);
            offset += dataChunkBytes[i].length;
            for (int j = 0; j < nodeHolderList.size(); j++) {
                nodeHolder = nodeHolderList.get(j);
                bufferSize = nodeHolder.getKeyLengths()[i] + (!nodeHolder.getIsSortedKeyBlock()[i] ? nodeHolder.getKeyBlockIndexLength()[i] : 0) + (dataWriterVo.getAggBlocks()[i] ? nodeHolder.getCompressedDataIndex()[i].length : 0);
                buffer = ByteBuffer.allocate(bufferSize);
                buffer.put(nodeHolder.getKeyArray()[i]);
                if (!nodeHolder.getIsSortedKeyBlock()[i]) {
                    buffer.putInt(nodeHolder.getCompressedIndex()[i].length);
                    buffer.put(nodeHolder.getCompressedIndex()[i]);
                    if (nodeHolder.getCompressedIndexMap()[i].length > 0) {
                        buffer.put(nodeHolder.getCompressedIndexMap()[i]);
                    }
                }
                if (nodeHolder.getAggBlocks()[i]) {
                    buffer.put(nodeHolder.getCompressedDataIndex()[i]);
                }
                buffer.flip();
                fileChannel.write(buffer);
                offset += bufferSize;
            }
        }
        dimensionOffset = offset;
        int dataChunkStartIndex = nodeHolderList.get(0).getKeyArray().length;
        for (int i = 0; i < numberOfMeasures; i++) {
            nodeHolderList = dataWriterHolder.getNodeHolder();
            currentDataChunksOffset.add(offset);
            currentDataChunksLength.add(dataChunkBytes[dataChunkStartIndex].length);
            buffer = ByteBuffer.allocate(dataChunkBytes[dataChunkStartIndex].length);
            buffer.put(dataChunkBytes[dataChunkStartIndex]);
            buffer.flip();
            fileChannel.write(buffer);
            offset += dataChunkBytes[dataChunkStartIndex].length;
            dataChunkStartIndex++;
            for (int j = 0; j < nodeHolderList.size(); j++) {
                nodeHolder = nodeHolderList.get(j);
                bufferSize = nodeHolder.getDataArray()[i].length;
                buffer = ByteBuffer.allocate(bufferSize);
                buffer.put(nodeHolder.getDataArray()[i]);
                buffer.flip();
                fileChannel.write(buffer);
                offset += bufferSize;
            }
        }
        measureOffset = offset;
    } catch (IOException e) {
        throw new CarbonDataWriterException("Problem while writing the data", e);
    }
    blockletIndex.add(CarbonMetadataUtil.getBlockletIndex(nodeHolderList, dataWriterVo.getSegmentProperties().getMeasures()));
    BlockletInfo3 blockletInfo3 = new BlockletInfo3(numberOfRows, currentDataChunksOffset, currentDataChunksLength, dimensionOffset, measureOffset, dataWriterHolder.getNodeHolder().size());
    blockletMetadata.add(blockletInfo3);
}

Example 2 with NodeHolder

use of org.apache.carbondata.core.util.NodeHolder in project carbondata by apache.

the class CarbonFactDataWriterImplV1 method buildDataNodeHolder.

@Override
public NodeHolder buildDataNodeHolder(IndexStorage<int[]>[] keyStorageArray, byte[][] measureArray, int entryCount, byte[] startKey, byte[] endKey, WriterCompressModel compressionModel, byte[] noDictionaryStartKey, byte[] noDictionaryEndKey, BitSet[] nullValueIndexBitSet) throws CarbonDataWriterException {
    // set the empty byte array
    if (null == noDictionaryEndKey) {
        noDictionaryEndKey = new byte[0];
    }
    if (null == noDictionaryStartKey) {
        noDictionaryStartKey = new byte[0];
    }
    // total measure length;
    int totalMsrArrySize = 0;
    // current measure length;
    int currentMsrLenght = 0;
    int totalKeySize = 0;
    int keyBlockSize = 0;
    boolean[] isSortedData = new boolean[keyStorageArray.length];
    int[] keyLengths = new int[keyStorageArray.length];
    //below will calculate min and max value for each column
    //for below 2d array, first index will be for column and second will be min max
    // value for same column
    //    byte[][] columnMinMaxData = new byte[keyStorageArray.length][];
    byte[][] allMinValue = new byte[keyStorageArray.length][];
    byte[][] allMaxValue = new byte[keyStorageArray.length][];
    byte[][] keyBlockData = fillAndCompressedKeyBlockData(keyStorageArray, entryCount);
    boolean[] colGrpBlock = new boolean[keyStorageArray.length];
    for (int i = 0; i < keyLengths.length; i++) {
        keyLengths[i] = keyBlockData[i].length;
        isSortedData[i] = keyStorageArray[i].isAlreadySorted();
        if (!isSortedData[i]) {
            keyBlockSize++;
        }
        totalKeySize += keyLengths[i];
        if (dataWriterVo.getIsComplexType()[i] || dataWriterVo.getIsDictionaryColumn()[i]) {
            allMinValue[i] = keyStorageArray[i].getMin();
            allMaxValue[i] = keyStorageArray[i].getMax();
        } else {
            allMinValue[i] = updateMinMaxForNoDictionary(keyStorageArray[i].getMin());
            allMaxValue[i] = updateMinMaxForNoDictionary(keyStorageArray[i].getMax());
        }
        //if keyStorageArray is instance of ColGroupBlockStorage than it's colGroup chunk
        if (keyStorageArray[i] instanceof ColGroupBlockStorage) {
            colGrpBlock[i] = true;
        }
    }
    int[] keyBlockIdxLengths = new int[keyBlockSize];
    byte[][] dataAfterCompression = new byte[keyBlockSize][];
    byte[][] indexMap = new byte[keyBlockSize][];
    int idx = 0;
    for (int i = 0; i < isSortedData.length; i++) {
        if (!isSortedData[i]) {
            dataAfterCompression[idx] = numberCompressor.compress(keyStorageArray[i].getDataAfterComp());
            if (null != keyStorageArray[i].getIndexMap() && keyStorageArray[i].getIndexMap().length > 0) {
                indexMap[idx] = numberCompressor.compress(keyStorageArray[i].getIndexMap());
            } else {
                indexMap[idx] = new byte[0];
            }
            keyBlockIdxLengths[idx] = (dataAfterCompression[idx].length + indexMap[idx].length) + CarbonCommonConstants.INT_SIZE_IN_BYTE;
            idx++;
        }
    }
    int compressDataBlockSize = 0;
    for (int i = 0; i < dataWriterVo.getAggBlocks().length; i++) {
        if (dataWriterVo.getAggBlocks()[i]) {
            compressDataBlockSize++;
        }
    }
    byte[][] compressedDataIndex = new byte[compressDataBlockSize][];
    int[] dataIndexMapLength = new int[compressDataBlockSize];
    idx = 0;
    for (int i = 0; i < dataWriterVo.getAggBlocks().length; i++) {
        if (dataWriterVo.getAggBlocks()[i]) {
            try {
                compressedDataIndex[idx] = numberCompressor.compress(keyStorageArray[i].getDataIndexMap());
                dataIndexMapLength[idx] = compressedDataIndex[idx].length;
                idx++;
            } catch (Exception e) {
                throw new CarbonDataWriterException(e.getMessage());
            }
        }
    }
    int[] msrLength = new int[dataWriterVo.getMeasureCount()];
    // each measure size
    for (int i = 0; i < measureArray.length; i++) {
        currentMsrLenght = measureArray[i].length;
        totalMsrArrySize += currentMsrLenght;
        msrLength[i] = currentMsrLenght;
    }
    NodeHolder holder = new NodeHolder();
    holder.setDataArray(measureArray);
    holder.setKeyArray(keyBlockData);
    holder.setMeasureNullValueIndex(nullValueIndexBitSet);
    // end key format will be <length of dictionary key><length of no
    // dictionary key><DictionaryKey><No Dictionary key>
    byte[] updatedNoDictionaryEndKey = updateNoDictionaryStartAndEndKey(noDictionaryEndKey);
    ByteBuffer buffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + CarbonCommonConstants.INT_SIZE_IN_BYTE + endKey.length + updatedNoDictionaryEndKey.length);
    buffer.putInt(endKey.length);
    buffer.putInt(updatedNoDictionaryEndKey.length);
    buffer.put(endKey);
    buffer.put(updatedNoDictionaryEndKey);
    buffer.rewind();
    holder.setEndKey(buffer.array());
    holder.setMeasureLenght(msrLength);
    byte[] updatedNoDictionaryStartKey = updateNoDictionaryStartAndEndKey(noDictionaryStartKey);
    // start key format will be <length of dictionary key><length of no
    // dictionary key><DictionaryKey><No Dictionary key>
    buffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + CarbonCommonConstants.INT_SIZE_IN_BYTE + startKey.length + updatedNoDictionaryStartKey.length);
    buffer.putInt(startKey.length);
    buffer.putInt(updatedNoDictionaryStartKey.length);
    buffer.put(startKey);
    buffer.put(updatedNoDictionaryStartKey);
    buffer.rewind();
    holder.setStartKey(buffer.array());
    holder.setEntryCount(entryCount);
    holder.setKeyLengths(keyLengths);
    holder.setKeyBlockIndexLength(keyBlockIdxLengths);
    holder.setIsSortedKeyBlock(isSortedData);
    holder.setCompressedIndex(dataAfterCompression);
    holder.setCompressedIndexMap(indexMap);
    holder.setDataIndexMapLength(dataIndexMapLength);
    holder.setCompressedDataIndex(compressedDataIndex);
    holder.setCompressionModel(compressionModel);
    holder.setTotalDimensionArrayLength(totalKeySize);
    holder.setTotalMeasureArrayLength(totalMsrArrySize);
    //setting column min max value
    holder.setColumnMaxData(allMaxValue);
    holder.setColumnMinData(allMinValue);
    holder.setAggBlocks(dataWriterVo.getAggBlocks());
    holder.setColGrpBlocks(colGrpBlock);
    return holder;
}

Example 3 with NodeHolder

use of org.apache.carbondata.core.util.NodeHolder in project carbondata by apache.

the class CarbonFactDataWriterImplV3 method writeDataToFile.

private void writeDataToFile(FileChannel channel) {
    // get the list of node holder list
    List<NodeHolder> nodeHolderList = dataWriterHolder.getNodeHolder();
    long blockletDataSize = 0;
    // get data chunks for all the column
    byte[][] dataChunkBytes = new byte[nodeHolderList.get(0).getKeyArray().length + nodeHolderList.get(0).getDataArray().length][];
    int measureStartIndex = nodeHolderList.get(0).getKeyArray().length;
    // calculate the size of data chunks
    try {
        for (int i = 0; i < nodeHolderList.get(0).getKeyArray().length; i++) {
            dataChunkBytes[i] = CarbonUtil.getByteArray(CarbonMetadataUtil.getDataChunk3(nodeHolderList, thriftColumnSchemaList, dataWriterVo.getSegmentProperties(), i, true));
            blockletDataSize += dataChunkBytes[i].length;
        }
        for (int i = 0; i < nodeHolderList.get(0).getDataArray().length; i++) {
            dataChunkBytes[measureStartIndex] = CarbonUtil.getByteArray(CarbonMetadataUtil.getDataChunk3(nodeHolderList, thriftColumnSchemaList, dataWriterVo.getSegmentProperties(), i, false));
            blockletDataSize += dataChunkBytes[measureStartIndex].length;
            measureStartIndex++;
        }
    } catch (IOException e) {
        throw new CarbonDataWriterException("Problem while getting the data chunks", e);
    }
    // calculate the total size of data to be written
    blockletDataSize += dataWriterHolder.getSize();
    // to check if data size will exceed the block size then create a new file
    updateBlockletFileChannel(blockletDataSize);
    // write data to file
    writeDataToFile(fileChannel, dataChunkBytes);
    // clear the data holder
    dataWriterHolder.clear();
}

Example 4 with NodeHolder

use of org.apache.carbondata.core.util.NodeHolder in project carbondata by apache.

the class CarbonFactDataWriterImplV3 method buildDataNodeHolder.

/**
   * Below method will be used to build the node holder object
   * This node holder object will be used to persist data which will
   * be written in carbon data file
   */
@Override
public NodeHolder buildDataNodeHolder(IndexStorage<short[]>[] keyStorageArray, byte[][] measureArray, int entryCount, byte[] startKey, byte[] endKey, WriterCompressModel compressionModel, byte[] noDictionaryStartKey, byte[] noDictionaryEndKey, BitSet[] nullValueIndexBitSet) throws CarbonDataWriterException {
    // set the empty byte array
    if (null == noDictionaryEndKey) {
        noDictionaryEndKey = new byte[0];
    }
    if (null == noDictionaryStartKey) {
        noDictionaryStartKey = new byte[0];
    }
    // total measure length;
    int totalMsrArrySize = 0;
    // current measure length;
    int currentMsrLenght = 0;
    int totalKeySize = 0;
    int keyBlockSize = 0;
    boolean[] isSortedData = new boolean[keyStorageArray.length];
    int[] keyLengths = new int[keyStorageArray.length];
    // below will calculate min and max value for each column
    // for below 2d array, first index will be for column and second will be min and max
    // value for same column
    byte[][] dimensionMinValue = new byte[keyStorageArray.length][];
    byte[][] dimensionMaxValue = new byte[keyStorageArray.length][];
    byte[][] measureMinValue = new byte[measureArray.length][];
    byte[][] measureMaxValue = new byte[measureArray.length][];
    byte[][] keyBlockData = fillAndCompressedKeyBlockData(keyStorageArray, entryCount);
    boolean[] colGrpBlock = new boolean[keyStorageArray.length];
    for (int i = 0; i < keyLengths.length; i++) {
        keyLengths[i] = keyBlockData[i].length;
        isSortedData[i] = keyStorageArray[i].isAlreadySorted();
        keyBlockSize++;
        totalKeySize += keyLengths[i];
        if (dataWriterVo.getIsComplexType()[i] || dataWriterVo.getIsDictionaryColumn()[i]) {
            dimensionMinValue[i] = keyStorageArray[i].getMin();
            dimensionMaxValue[i] = keyStorageArray[i].getMax();
        } else {
            dimensionMinValue[i] = updateMinMaxForNoDictionary(keyStorageArray[i].getMin());
            dimensionMaxValue[i] = updateMinMaxForNoDictionary(keyStorageArray[i].getMax());
        }
        // colGroup chunk
        if (keyStorageArray[i] instanceof ColGroupBlockStorage) {
            colGrpBlock[i] = true;
        }
    }
    for (int i = 0; i < measureArray.length; i++) {
        measureMaxValue[i] = CarbonMetadataUtil.getByteValueForMeasure(compressionModel.getMaxValue()[i], dataWriterVo.getSegmentProperties().getMeasures().get(i).getDataType());
        measureMinValue[i] = CarbonMetadataUtil.getByteValueForMeasure(compressionModel.getMinValue()[i], dataWriterVo.getSegmentProperties().getMeasures().get(i).getDataType());
    }
    int[] keyBlockIdxLengths = new int[keyBlockSize];
    byte[][] dataAfterCompression = new byte[keyBlockSize][];
    byte[][] indexMap = new byte[keyBlockSize][];
    for (int i = 0; i < isSortedData.length; i++) {
        if (!isSortedData[i]) {
            dataAfterCompression[i] = getByteArray(keyStorageArray[i].getDataAfterComp());
            if (null != keyStorageArray[i].getIndexMap() && keyStorageArray[i].getIndexMap().length > 0) {
                indexMap[i] = getByteArray(keyStorageArray[i].getIndexMap());
            } else {
                indexMap[i] = new byte[0];
            }
            keyBlockIdxLengths[i] = (dataAfterCompression[i].length + indexMap[i].length) + CarbonCommonConstants.INT_SIZE_IN_BYTE;
        }
    }
    byte[][] compressedDataIndex = new byte[keyBlockSize][];
    int[] dataIndexMapLength = new int[keyBlockSize];
    for (int i = 0; i < dataWriterVo.getAggBlocks().length; i++) {
        if (dataWriterVo.getAggBlocks()[i]) {
            try {
                compressedDataIndex[i] = getByteArray(keyStorageArray[i].getDataIndexMap());
                dataIndexMapLength[i] = compressedDataIndex[i].length;
            } catch (Exception e) {
                throw new CarbonDataWriterException(e.getMessage(), e);
            }
        }
    }
    int[] msrLength = new int[dataWriterVo.getMeasureCount()];
    // each measure size
    for (int i = 0; i < measureArray.length; i++) {
        currentMsrLenght = measureArray[i].length;
        totalMsrArrySize += currentMsrLenght;
        msrLength[i] = currentMsrLenght;
    }
    NodeHolder holder = new NodeHolder();
    holder.setDataArray(measureArray);
    holder.setKeyArray(keyBlockData);
    holder.setMeasureNullValueIndex(nullValueIndexBitSet);
    // end key format will be <length of dictionary key><length of no
    // dictionary key><DictionaryKey><No Dictionary key>
    byte[] updatedNoDictionaryEndKey = updateNoDictionaryStartAndEndKey(noDictionaryEndKey);
    ByteBuffer buffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + CarbonCommonConstants.INT_SIZE_IN_BYTE + endKey.length + updatedNoDictionaryEndKey.length);
    buffer.putInt(endKey.length);
    buffer.putInt(updatedNoDictionaryEndKey.length);
    buffer.put(endKey);
    buffer.put(updatedNoDictionaryEndKey);
    buffer.rewind();
    holder.setEndKey(buffer.array());
    holder.setMeasureLenght(msrLength);
    byte[] updatedNoDictionaryStartKey = updateNoDictionaryStartAndEndKey(noDictionaryStartKey);
    // start key format will be <length of dictionary key><length of no
    // dictionary key><DictionaryKey><No Dictionary key>
    buffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + CarbonCommonConstants.INT_SIZE_IN_BYTE + startKey.length + updatedNoDictionaryStartKey.length);
    buffer.putInt(startKey.length);
    buffer.putInt(updatedNoDictionaryStartKey.length);
    buffer.put(startKey);
    buffer.put(updatedNoDictionaryStartKey);
    buffer.rewind();
    holder.setStartKey(buffer.array());
    holder.setEntryCount(entryCount);
    holder.setKeyLengths(keyLengths);
    holder.setKeyBlockIndexLength(keyBlockIdxLengths);
    holder.setIsSortedKeyBlock(isSortedData);
    holder.setCompressedIndex(dataAfterCompression);
    holder.setCompressedIndexMap(indexMap);
    holder.setDataIndexMapLength(dataIndexMapLength);
    holder.setCompressedDataIndex(compressedDataIndex);
    holder.setCompressionModel(compressionModel);
    holder.setTotalDimensionArrayLength(totalKeySize);
    holder.setTotalMeasureArrayLength(totalMsrArrySize);
    holder.setMeasureColumnMaxData(measureMaxValue);
    holder.setMeasureColumnMinData(measureMinValue);
    // setting column min max value
    holder.setColumnMaxData(dimensionMaxValue);
    holder.setColumnMinData(dimensionMinValue);
    holder.setAggBlocks(dataWriterVo.getAggBlocks());
    holder.setColGrpBlocks(colGrpBlock);
    List<byte[]> dimensionDataChunk2 = null;
    List<byte[]> measureDataChunk2 = null;
    try {
        dimensionDataChunk2 = CarbonMetadataUtil.getDataChunk2(holder, thriftColumnSchemaList, dataWriterVo.getSegmentProperties(), true);
        measureDataChunk2 = CarbonMetadataUtil.getDataChunk2(holder, thriftColumnSchemaList, dataWriterVo.getSegmentProperties(), false);
    } catch (IOException e) {
        throw new CarbonDataWriterException(e.getMessage());
    }
    holder.setHolderSize(calculateSize(holder, dimensionDataChunk2, measureDataChunk2));
    return holder;
}