Examples with IndexRowImpl org.apache.carbondata.core.indexstore.row.IndexRowImpl used on opensource projects

Example 6 with IndexRowImpl

use of org.apache.carbondata.core.indexstore.row.IndexRowImpl in project carbondata by apache.

the class BlockIndex method addTaskMinMaxValues.

/**
 * This method will compute min/max values at task level
 *
 * @param taskMinMaxRow
 * @param carbonRowSchema
 * @param taskMinMaxOrdinal
 * @param minMaxValue
 * @param ordinal
 * @param isMinValueComparison
 */
protected void addTaskMinMaxValues(IndexRow taskMinMaxRow, CarbonRowSchema[] carbonRowSchema, int taskMinMaxOrdinal, byte[][] minMaxValue, int ordinal, boolean isMinValueComparison, List<ColumnSchema> columnSchemaList) {
    IndexRow row = taskMinMaxRow.getRow(ordinal);
    byte[][] updatedMinMaxValues = null;
    if (null == row) {
        CarbonRowSchema[] minSchemas = ((CarbonRowSchema.StructCarbonRowSchema) carbonRowSchema[taskMinMaxOrdinal]).getChildSchemas();
        row = new IndexRowImpl(minSchemas);
        updatedMinMaxValues = minMaxValue;
    } else {
        byte[][] existingMinMaxValues = getMinMaxValue(taskMinMaxRow, ordinal);
        updatedMinMaxValues = compareAndUpdateMinMax(minMaxValue, existingMinMaxValues, isMinValueComparison, columnSchemaList);
    }
    int minMaxOrdinal = 0;
    // min/max value adding
    for (int i = 0; i < updatedMinMaxValues.length; i++) {
        row.setByteArray(updatedMinMaxValues[i], minMaxOrdinal++);
    }
    taskMinMaxRow.setRow(row, ordinal);
}

Example 7 with IndexRowImpl

use of org.apache.carbondata.core.indexstore.row.IndexRowImpl in project carbondata by apache.

the class BlockletIndex method loadToUnsafe.

private IndexRowImpl loadToUnsafe(CarbonRowSchema[] schema, CarbonRowSchema[] taskSummarySchema, DataFileFooter fileFooter, SegmentProperties segmentProperties, List<CarbonColumn> minMaxCacheColumns, String filePath, IndexRowImpl summaryRow, BlockMetaInfo blockMetaInfo, int relativeBlockletId) {
    List<BlockletInfo> blockletList = fileFooter.getBlockletList();
    // Add one row to maintain task level min max for segment pruning
    if (!blockletList.isEmpty() && summaryRow == null) {
        summaryRow = new IndexRowImpl(taskSummarySchema);
    }
    for (int index = 0; index < blockletList.size(); index++) {
        IndexRow row = new IndexRowImpl(schema);
        int ordinal = 0;
        int taskMinMaxOrdinal = 1;
        BlockletInfo blockletInfo = blockletList.get(index);
        blockletInfo.setSorted(fileFooter.isSorted());
        BlockletMinMaxIndex minMaxIndex = blockletInfo.getBlockletIndex().getMinMaxIndex();
        // get min max values for columns to be cached
        byte[][] minValuesForColumnsToBeCached = BlockletIndexUtil.getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns, minMaxIndex.getMinValues());
        byte[][] maxValuesForColumnsToBeCached = BlockletIndexUtil.getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns, minMaxIndex.getMaxValues());
        boolean[] minMaxFlagValuesForColumnsToBeCached = BlockletIndexUtil.getMinMaxFlagValuesForColumnsToBeCached(segmentProperties, minMaxCacheColumns, fileFooter.getBlockletIndex().getMinMaxIndex().getIsMinMaxSet());
        row.setRow(addMinMax(schema[ordinal], minValuesForColumnsToBeCached), ordinal);
        // compute and set task level min values
        addTaskMinMaxValues(summaryRow, taskSummarySchema, taskMinMaxOrdinal, minValuesForColumnsToBeCached, TASK_MIN_VALUES_INDEX, true, fileFooter.getColumnInTable());
        ordinal++;
        taskMinMaxOrdinal++;
        row.setRow(addMinMax(schema[ordinal], maxValuesForColumnsToBeCached), ordinal);
        // compute and set task level max values
        addTaskMinMaxValues(summaryRow, taskSummarySchema, taskMinMaxOrdinal, maxValuesForColumnsToBeCached, TASK_MAX_VALUES_INDEX, false, fileFooter.getColumnInTable());
        ordinal++;
        row.setInt(blockletInfo.getNumberOfRows(), ordinal++);
        // add file name
        byte[] filePathBytes = CarbonTablePath.getCarbonDataFileName(filePath).getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
        row.setByteArray(filePathBytes, ordinal++);
        // add version number
        row.setShort(fileFooter.getVersionId().number(), ordinal++);
        // add schema updated time
        row.setLong(fileFooter.getSchemaUpdatedTimeStamp(), ordinal++);
        byte[] serializedData;
        try {
            // Add block footer offset, it is used if we need to read footer of block
            row.setLong(fileFooter.getBlockInfo().getBlockOffset(), ordinal++);
            setLocations(blockMetaInfo.getLocationInfo(), row, ordinal++);
            // Store block size
            row.setLong(blockMetaInfo.getSize(), ordinal++);
            // add min max flag for all the dimension columns
            addMinMaxFlagValues(row, schema[ordinal], minMaxFlagValuesForColumnsToBeCached, ordinal);
            ordinal++;
            // add blocklet info
            ByteArrayOutputStream stream = new ByteArrayOutputStream();
            DataOutput dataOutput = new DataOutputStream(stream);
            blockletInfo.write(dataOutput);
            serializedData = stream.toByteArray();
            row.setByteArray(serializedData, ordinal++);
            // add pages
            row.setShort((short) blockletInfo.getNumberOfPages(), ordinal++);
            // for relative blocklet id i.e blocklet id that belongs to a particular carbondata file
            row.setShort((short) relativeBlockletId++, ordinal);
            memoryDMStore.addIndexRow(schema, row);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    return summaryRow;
}

Example 8 with IndexRowImpl

use of org.apache.carbondata.core.indexstore.row.IndexRowImpl in project carbondata by apache.

the class BlockIndex method loadToUnsafeBlock.

/**
 * Load information for the block.It is the case can happen only for old stores
 * where blocklet information is not available in index file. So load only block information
 * and read blocklet information in executor.
 */
protected IndexRowImpl loadToUnsafeBlock(CarbonRowSchema[] schema, CarbonRowSchema[] taskSummarySchema, DataFileFooter fileFooter, SegmentProperties segmentProperties, List<CarbonColumn> minMaxCacheColumns, String filePath, IndexRowImpl summaryRow, BlockMetaInfo blockMetaInfo, byte[][] minValues, byte[][] maxValues, boolean[] minMaxFlag) {
    // Add one row to maintain task level min max for segment pruning
    if (summaryRow == null) {
        summaryRow = new IndexRowImpl(taskSummarySchema);
    }
    IndexRow row = new IndexRowImpl(schema);
    int ordinal = 0;
    int taskMinMaxOrdinal = 1;
    // get min max values for columns to be cached
    byte[][] minValuesForColumnsToBeCached = BlockletIndexUtil.getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns, minValues);
    byte[][] maxValuesForColumnsToBeCached = BlockletIndexUtil.getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns, maxValues);
    boolean[] minMaxFlagValuesForColumnsToBeCached = BlockletIndexUtil.getMinMaxFlagValuesForColumnsToBeCached(segmentProperties, minMaxCacheColumns, minMaxFlag);
    IndexRow indexRow = addMinMax(schema[ordinal], minValuesForColumnsToBeCached);
    row.setRow(indexRow, ordinal);
    // compute and set task level min values
    addTaskMinMaxValues(summaryRow, taskSummarySchema, taskMinMaxOrdinal, minValuesForColumnsToBeCached, TASK_MIN_VALUES_INDEX, true, fileFooter.getColumnInTable());
    ordinal++;
    taskMinMaxOrdinal++;
    row.setRow(addMinMax(schema[ordinal], maxValuesForColumnsToBeCached), ordinal);
    // compute and set task level max values
    addTaskMinMaxValues(summaryRow, taskSummarySchema, taskMinMaxOrdinal, maxValuesForColumnsToBeCached, TASK_MAX_VALUES_INDEX, false, fileFooter.getColumnInTable());
    ordinal++;
    // add total rows in one carbondata file
    row.setInt((int) fileFooter.getNumberOfRows(), ordinal++);
    // add file name
    byte[] filePathBytes = CarbonTablePath.getCarbonDataFileName(filePath).getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
    row.setByteArray(filePathBytes, ordinal++);
    // add version number
    row.setShort(fileFooter.getVersionId().number(), ordinal++);
    // add schema updated time
    row.setLong(fileFooter.getSchemaUpdatedTimeStamp(), ordinal++);
    // add block offset
    row.setLong(fileFooter.getBlockInfo().getBlockOffset(), ordinal++);
    try {
        setLocations(blockMetaInfo.getLocationInfo(), row, ordinal++);
        // store block size
        row.setLong(blockMetaInfo.getSize(), ordinal++);
        // add min max flag for all the dimension columns
        addMinMaxFlagValues(row, schema[ordinal], minMaxFlagValuesForColumnsToBeCached, ordinal);
        memoryDMStore.addIndexRow(schema, row);
    } catch (Exception e) {
        String message = "Load to unsafe failed for block: " + filePath;
        LOGGER.error(message, e);
        throw new RuntimeException(message, e);
    }
    return summaryRow;
}