Examples with BlockletInfo org.apache.carbondata.core.metadata.blocklet.BlockletInfo used on opensource projects

Example 11 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class CarbonFactDataWriterImplV3 method fillBlockIndexInfoDetails.

/**
 * Below method will be used to fill the block info details
 *
 * @param numberOfRows       number of rows in file
 * @param carbonDataFileName The name of carbonData file
 * @param currentPosition    current offset
 */
@Override
protected void fillBlockIndexInfoDetails(long numberOfRows, String carbonDataFileName, long currentPosition) {
    int i = 0;
    DataFileFooterConverterV3 converterV3 = new DataFileFooterConverterV3();
    for (org.apache.carbondata.format.BlockletIndex index : blockletIndex) {
        BlockletInfo3 blockletInfo3 = blockletMetadata.get(i);
        BlockletInfo blockletInfo = converterV3.getBlockletInfo(blockletInfo3, model.getSegmentProperties().getDimensions().size());
        BlockletBTreeIndex bTreeIndex = new BlockletBTreeIndex(index.b_tree_index.getStart_key(), index.b_tree_index.getEnd_key());
        BlockletMinMaxIndex minMaxIndex = new BlockletMinMaxIndex();
        minMaxIndex.setMinValues(toByteArray(index.getMin_max_index().getMin_values()));
        minMaxIndex.setMaxValues(toByteArray(index.getMin_max_index().getMax_values()));
        org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex bIndex = new org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex(bTreeIndex, minMaxIndex);
        BlockIndexInfo biInfo = new BlockIndexInfo(numberOfRows, carbonDataFileName, currentPosition, bIndex, blockletInfo);
        blockIndexInfoList.add(biInfo);
        i++;
    }
}

Example 12 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class SegmentTaskIndexStoreTest method getDataFileFooters.

private List<DataFileFooter> getDataFileFooters() {
    SegmentInfo segmentInfo = new SegmentInfo();
    DataFileFooter footer = new DataFileFooter();
    ColumnSchema columnSchema = new ColumnSchema();
    BlockletInfo blockletInfo = new BlockletInfo();
    List<DataFileFooter> footerList = new ArrayList<DataFileFooter>();
    List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
    columnSchema.setColumnName("employeeName");
    columnSchemaList.add(new ColumnSchema());
    footer.setSegmentInfo(segmentInfo);
    footer.setColumnInTable(columnSchemaList);
    footer.setBlockletList(Arrays.asList(blockletInfo));
    footerList.add(footer);
    return footerList;
}

Example 13 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class DataFileFooterConverter2 method getBlockletInfo.

/**
 * Below method is to convert the blocklet info of the thrift to wrapper
 * blocklet info
 *
 * @param blockletInfoThrift blocklet info of the thrift
 * @return blocklet info wrapper
 */
private BlockletInfo getBlockletInfo(org.apache.carbondata.format.BlockletInfo2 blockletInfoThrift, int numberOfDimensionColumns) {
    BlockletInfo blockletInfo = new BlockletInfo();
    List<Long> dimensionColumnChunkOffsets = blockletInfoThrift.getColumn_data_chunks_offsets().subList(0, numberOfDimensionColumns);
    List<Long> measureColumnChunksOffsets = blockletInfoThrift.getColumn_data_chunks_offsets().subList(numberOfDimensionColumns, blockletInfoThrift.getColumn_data_chunks_offsets().size());
    List<Short> dimensionColumnChunkLength = blockletInfoThrift.getColumn_data_chunks_length().subList(0, numberOfDimensionColumns);
    List<Short> measureColumnChunksLength = blockletInfoThrift.getColumn_data_chunks_length().subList(numberOfDimensionColumns, blockletInfoThrift.getColumn_data_chunks_offsets().size());
    blockletInfo.setDimensionChunkOffsets(dimensionColumnChunkOffsets);
    blockletInfo.setMeasureChunkOffsets(measureColumnChunksOffsets);
    List<Integer> dimensionColumnChunkLengthInteger = new ArrayList<Integer>();
    List<Integer> measureColumnChunkLengthInteger = new ArrayList<Integer>();
    for (int i = 0; i < dimensionColumnChunkLength.size(); i++) {
        dimensionColumnChunkLengthInteger.add(dimensionColumnChunkLength.get(i).intValue());
    }
    for (int i = 0; i < measureColumnChunksLength.size(); i++) {
        measureColumnChunkLengthInteger.add(measureColumnChunksLength.get(i).intValue());
    }
    blockletInfo.setDimensionChunksLength(dimensionColumnChunkLengthInteger);
    blockletInfo.setMeasureChunksLength(measureColumnChunkLengthInteger);
    blockletInfo.setNumberOfRows(blockletInfoThrift.getNum_rows());
    return blockletInfo;
}

Example 14 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class BlockletDataMap method createBlocklet.

private ExtendedBlocklet createBlocklet(DataMapRow row, int blockletId) {
    ExtendedBlocklet blocklet = new ExtendedBlocklet(new String(row.getByteArray(FILE_PATH_INDEX), CarbonCommonConstants.DEFAULT_CHARSET_CLASS), blockletId + "");
    BlockletDetailInfo detailInfo = new BlockletDetailInfo();
    detailInfo.setRowCount(row.getInt(ROW_COUNT_INDEX));
    detailInfo.setPagesCount(row.getShort(PAGE_COUNT_INDEX));
    detailInfo.setVersionNumber(row.getShort(VERSION_INDEX));
    detailInfo.setBlockletId((short) blockletId);
    detailInfo.setDimLens(columnCardinality);
    detailInfo.setSchemaUpdatedTimeStamp(row.getLong(SCHEMA_UPADATED_TIME_INDEX));
    byte[] byteArray = row.getByteArray(BLOCK_INFO_INDEX);
    BlockletInfo blockletInfo = null;
    try {
        if (byteArray.length > 0) {
            blockletInfo = new BlockletInfo();
            ByteArrayInputStream stream = new ByteArrayInputStream(byteArray);
            DataInputStream inputStream = new DataInputStream(stream);
            blockletInfo.readFields(inputStream);
            inputStream.close();
        }
        blocklet.setLocation(new String(row.getByteArray(LOCATIONS), CarbonCommonConstants.DEFAULT_CHARSET).split(","));
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    detailInfo.setBlockletInfo(blockletInfo);
    blocklet.setDetailInfo(detailInfo);
    detailInfo.setBlockFooterOffset(row.getLong(BLOCK_FOOTER_OFFSET));
    detailInfo.setColumnSchemaBinary(getColumnSchemaBinary());
    detailInfo.setBlockSize(row.getLong(BLOCK_LENGTH));
    return blocklet;
}

Example 15 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class BlockletDataMap method loadToUnsafe.

private DataMapRowImpl loadToUnsafe(DataFileFooter fileFooter, SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow, BlockMetaInfo blockMetaInfo, int relativeBlockletId) {
    int[] minMaxLen = segmentProperties.getColumnsValueSize();
    List<BlockletInfo> blockletList = fileFooter.getBlockletList();
    CarbonRowSchema[] schema = unsafeMemoryDMStore.getSchema();
    // Add one row to maintain task level min max for segment pruning
    if (!blockletList.isEmpty() && summaryRow == null) {
        summaryRow = new DataMapRowImpl(unsafeMemorySummaryDMStore.getSchema());
    }
    for (int index = 0; index < blockletList.size(); index++) {
        DataMapRow row = new DataMapRowImpl(schema);
        int ordinal = 0;
        int taskMinMaxOrdinal = 0;
        BlockletInfo blockletInfo = blockletList.get(index);
        // add start key as index key
        row.setByteArray(blockletInfo.getBlockletIndex().getBtreeIndex().getStartKey(), ordinal++);
        BlockletMinMaxIndex minMaxIndex = blockletInfo.getBlockletIndex().getMinMaxIndex();
        byte[][] minValues = updateMinValues(minMaxIndex.getMinValues(), minMaxLen);
        row.setRow(addMinMax(minMaxLen, schema[ordinal], minValues), ordinal);
        // compute and set task level min values
        addTaskMinMaxValues(summaryRow, minMaxLen, unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], minValues, TASK_MIN_VALUES_INDEX, true);
        ordinal++;
        taskMinMaxOrdinal++;
        byte[][] maxValues = updateMaxValues(minMaxIndex.getMaxValues(), minMaxLen);
        row.setRow(addMinMax(minMaxLen, schema[ordinal], maxValues), ordinal);
        // compute and set task level max values
        addTaskMinMaxValues(summaryRow, minMaxLen, unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], maxValues, TASK_MAX_VALUES_INDEX, false);
        ordinal++;
        row.setInt(blockletInfo.getNumberOfRows(), ordinal++);
        // add file path
        byte[] filePathBytes = filePath.getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
        row.setByteArray(filePathBytes, ordinal++);
        // add pages
        row.setShort((short) blockletInfo.getNumberOfPages(), ordinal++);
        // add version number
        row.setShort(fileFooter.getVersionId().number(), ordinal++);
        // add schema updated time
        row.setLong(fileFooter.getSchemaUpdatedTimeStamp(), ordinal++);
        // add blocklet info
        byte[] serializedData;
        try {
            ByteArrayOutputStream stream = new ByteArrayOutputStream();
            DataOutput dataOutput = new DataOutputStream(stream);
            blockletInfo.write(dataOutput);
            serializedData = stream.toByteArray();
            row.setByteArray(serializedData, ordinal++);
            // Add block footer offset, it is used if we need to read footer of block
            row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);
            setLocations(blockMetaInfo.getLocationInfo(), row, ordinal);
            ordinal++;
            // for relative blockelt id i.e blocklet id that belongs to a particular part file
            row.setShort((short) relativeBlockletId++, ordinal++);
            // Store block size
            row.setLong(blockMetaInfo.getSize(), ordinal);
            unsafeMemoryDMStore.addIndexRowToUnsafe(row);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    return summaryRow;
}