Examples with CarbonRowSchema org.apache.carbondata.core.indexstore.schema.CarbonRowSchema used on opensource projects

Example 6 with CarbonRowSchema

use of org.apache.carbondata.core.indexstore.schema.CarbonRowSchema in project carbondata by apache.

the class BlockIndex method addMinMaxFlagValues.

protected void addMinMaxFlagValues(IndexRow row, CarbonRowSchema carbonRowSchema, boolean[] minMaxFlag, int ordinal) {
    CarbonRowSchema[] minMaxFlagSchema = ((CarbonRowSchema.StructCarbonRowSchema) carbonRowSchema).getChildSchemas();
    IndexRow minMaxFlagRow = new IndexRowImpl(minMaxFlagSchema);
    int flagOrdinal = 0;
    // min value adding
    for (int i = 0; i < minMaxFlag.length; i++) {
        minMaxFlagRow.setBoolean(minMaxFlag[i], flagOrdinal++);
    }
    row.setRow(minMaxFlagRow, ordinal);
}

Example 7 with CarbonRowSchema

use of org.apache.carbondata.core.indexstore.schema.CarbonRowSchema in project carbondata by apache.

the class BlockIndex method getRowCountForEachBlock.

public Map<String, Long> getRowCountForEachBlock(Segment segment, List<PartitionSpec> partitions, Map<String, Long> blockletToRowCountMap) {
    if (memoryDMStore.getRowCount() == 0) {
        return new HashMap<>();
    }
    CarbonRowSchema[] schema = getFileFooterEntrySchema();
    int numEntries = memoryDMStore.getRowCount();
    for (int i = 0; i < numEntries; i++) {
        IndexRow indexRow = memoryDMStore.getIndexRow(schema, i);
        String fileName = new String(indexRow.getByteArray(FILE_PATH_INDEX), CarbonCommonConstants.DEFAULT_CHARSET_CLASS) + CarbonTablePath.getCarbonDataExtension();
        int rowCount = indexRow.getInt(ROW_COUNT_INDEX);
        // prepend segment number with the blocklet file path
        String blockletMapKey = segment.getSegmentNo() + "," + fileName;
        Long existingCount = blockletToRowCountMap.get(blockletMapKey);
        if (null != existingCount) {
            blockletToRowCountMap.put(blockletMapKey, (long) rowCount + existingCount);
        } else {
            blockletToRowCountMap.put(blockletMapKey, (long) rowCount);
        }
    }
    return blockletToRowCountMap;
}

Example 8 with CarbonRowSchema

use of org.apache.carbondata.core.indexstore.schema.CarbonRowSchema in project carbondata by apache.

the class BlockIndex method init.

@Override
public void init(IndexModel indexModel) throws IOException {
    long startTime = System.currentTimeMillis();
    assert (indexModel instanceof BlockletIndexModel);
    BlockletIndexModel blockletIndexModel = (BlockletIndexModel) indexModel;
    DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter(indexModel.getConfiguration());
    List<DataFileFooter> indexInfo = null;
    if (blockletIndexModel.getIndexInfos() == null || blockletIndexModel.getIndexInfos().isEmpty()) {
        indexInfo = fileFooterConverter.getIndexInfo(blockletIndexModel.getFilePath(), blockletIndexModel.getFileData(), blockletIndexModel.getCarbonTable().isTransactionalTable());
    } else {
        // when index info is already read and converted to data file footer object
        indexInfo = blockletIndexModel.getIndexInfos();
    }
    String path = blockletIndexModel.getFilePath();
    // store file path only in case of partition table, non transactional table and flat folder
    // structure
    byte[] filePath;
    this.isPartitionTable = blockletIndexModel.getCarbonTable().isHivePartitionTable();
    if (this.isPartitionTable || !blockletIndexModel.getCarbonTable().isTransactionalTable() || blockletIndexModel.getCarbonTable().isSupportFlatFolder() || // if the segment data is written in table path then no need to store whole path of file.
    !blockletIndexModel.getFilePath().startsWith(blockletIndexModel.getCarbonTable().getTablePath())) {
        filePath = FilenameUtils.getFullPathNoEndSeparator(path).getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
        isFilePathStored = true;
    } else {
        filePath = new byte[0];
    }
    byte[] fileName = path.substring(path.lastIndexOf("/") + 1).getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
    byte[] segmentId = blockletIndexModel.getSegmentId().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
    if (!indexInfo.isEmpty()) {
        DataFileFooter fileFooter = indexInfo.get(0);
        // init segment properties and create schema
        SegmentProperties segmentProperties = initSegmentProperties(blockletIndexModel, fileFooter);
        createMemorySchema(blockletIndexModel);
        createSummaryDMStore(blockletIndexModel);
        CarbonRowSchema[] taskSummarySchema = getTaskSummarySchema();
        // check for legacy store and load the metadata
        IndexRowImpl summaryRow = loadMetadata(taskSummarySchema, segmentProperties, blockletIndexModel, indexInfo);
        finishWriting(taskSummarySchema, filePath, fileName, segmentId, summaryRow);
        if (((BlockletIndexModel) indexModel).isSerializeDmStore()) {
            serializeDmStore();
        }
    }
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("Time taken to load blocklet index from file : " + indexModel.getFilePath() + " is " + (System.currentTimeMillis() - startTime));
    }
}

Example 9 with CarbonRowSchema

use of org.apache.carbondata.core.indexstore.schema.CarbonRowSchema in project carbondata by apache.

the class UnsafeMemoryDMStore method addToUnsafe.

private void addToUnsafe(CarbonRowSchema schema, DataMapRow row, int index) {
    switch(schema.getSchemaType()) {
        case FIXED:
            DataType dataType = schema.getDataType();
            if (dataType == DataTypes.BYTE) {
                getUnsafe().putByte(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getByte(index));
                runningLength += row.getSizeInBytes(index);
            } else if (dataType == DataTypes.SHORT) {
                getUnsafe().putShort(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getShort(index));
                runningLength += row.getSizeInBytes(index);
            } else if (dataType == DataTypes.INT) {
                getUnsafe().putInt(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getInt(index));
                runningLength += row.getSizeInBytes(index);
            } else if (dataType == DataTypes.LONG) {
                getUnsafe().putLong(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getLong(index));
                runningLength += row.getSizeInBytes(index);
            } else if (dataType == DataTypes.FLOAT) {
                getUnsafe().putFloat(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getFloat(index));
                runningLength += row.getSizeInBytes(index);
            } else if (dataType == DataTypes.DOUBLE) {
                getUnsafe().putDouble(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getDouble(index));
                runningLength += row.getSizeInBytes(index);
            } else if (dataType == DataTypes.BYTE_ARRAY) {
                byte[] data = row.getByteArray(index);
                getUnsafe().copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, data.length);
                runningLength += row.getSizeInBytes(index);
            } else {
                throw new UnsupportedOperationException("unsupported data type for unsafe storage: " + schema.getDataType());
            }
            break;
        case VARIABLE:
            byte[] data = row.getByteArray(index);
            getUnsafe().putShort(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, (short) data.length);
            runningLength += 2;
            getUnsafe().copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, data.length);
            runningLength += data.length;
            break;
        case STRUCT:
            CarbonRowSchema[] childSchemas = ((CarbonRowSchema.StructCarbonRowSchema) schema).getChildSchemas();
            DataMapRow struct = row.getRow(index);
            for (int i = 0; i < childSchemas.length; i++) {
                addToUnsafe(childSchemas[i], struct, i);
            }
            break;
        default:
            throw new UnsupportedOperationException("unsupported data type for unsafe storage: " + schema.getDataType());
    }
}

Example 10 with CarbonRowSchema

use of org.apache.carbondata.core.indexstore.schema.CarbonRowSchema in project carbondata by apache.

the class BlockletDataMap method addTaskMinMaxValues.

/**
 * This method will compute min/max values at task level
 *
 * @param taskMinMaxRow
 * @param minMaxLen
 * @param carbonRowSchema
 * @param minMaxValue
 * @param ordinal
 * @param isMinValueComparison
 */
private void addTaskMinMaxValues(DataMapRow taskMinMaxRow, int[] minMaxLen, CarbonRowSchema carbonRowSchema, byte[][] minMaxValue, int ordinal, boolean isMinValueComparison) {
    DataMapRow row = taskMinMaxRow.getRow(ordinal);
    byte[][] updatedMinMaxValues = minMaxValue;
    if (null == row) {
        CarbonRowSchema[] minSchemas = ((CarbonRowSchema.StructCarbonRowSchema) carbonRowSchema).getChildSchemas();
        row = new DataMapRowImpl(minSchemas);
    } else {
        byte[][] existingMinMaxValues = getMinMaxValue(taskMinMaxRow, ordinal);
        // Compare and update min max values
        for (int i = 0; i < minMaxLen.length; i++) {
            int compare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(existingMinMaxValues[i], minMaxValue[i]);
            if (isMinValueComparison) {
                if (compare < 0) {
                    updatedMinMaxValues[i] = existingMinMaxValues[i];
                }
            } else if (compare > 0) {
                updatedMinMaxValues[i] = existingMinMaxValues[i];
            }
        }
    }
    int minMaxOrdinal = 0;
    // min/max value adding
    for (int i = 0; i < minMaxLen.length; i++) {
        row.setByteArray(updatedMinMaxValues[i], minMaxOrdinal++);
    }
    taskMinMaxRow.setRow(row, ordinal);
}