Examples with DataMapRow org.apache.carbondata.core.indexstore.row.DataMapRow used on opensource projects

Example 6 with DataMapRow

use of org.apache.carbondata.core.indexstore.row.DataMapRow in project carbondata by apache.

the class UnsafeMemoryDMStore method addToUnsafe.

private void addToUnsafe(CarbonRowSchema schema, DataMapRow row, int index) {
    switch(schema.getSchemaType()) {
        case FIXED:
            DataType dataType = schema.getDataType();
            if (dataType == DataTypes.BYTE) {
                getUnsafe().putByte(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getByte(index));
                runningLength += row.getSizeInBytes(index);
            } else if (dataType == DataTypes.SHORT) {
                getUnsafe().putShort(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getShort(index));
                runningLength += row.getSizeInBytes(index);
            } else if (dataType == DataTypes.INT) {
                getUnsafe().putInt(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getInt(index));
                runningLength += row.getSizeInBytes(index);
            } else if (dataType == DataTypes.LONG) {
                getUnsafe().putLong(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getLong(index));
                runningLength += row.getSizeInBytes(index);
            } else if (dataType == DataTypes.FLOAT) {
                getUnsafe().putFloat(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getFloat(index));
                runningLength += row.getSizeInBytes(index);
            } else if (dataType == DataTypes.DOUBLE) {
                getUnsafe().putDouble(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getDouble(index));
                runningLength += row.getSizeInBytes(index);
            } else if (dataType == DataTypes.BYTE_ARRAY) {
                byte[] data = row.getByteArray(index);
                getUnsafe().copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, data.length);
                runningLength += row.getSizeInBytes(index);
            } else {
                throw new UnsupportedOperationException("unsupported data type for unsafe storage: " + schema.getDataType());
            }
            break;
        case VARIABLE:
            byte[] data = row.getByteArray(index);
            getUnsafe().putShort(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, (short) data.length);
            runningLength += 2;
            getUnsafe().copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, data.length);
            runningLength += data.length;
            break;
        case STRUCT:
            CarbonRowSchema[] childSchemas = ((CarbonRowSchema.StructCarbonRowSchema) schema).getChildSchemas();
            DataMapRow struct = row.getRow(index);
            for (int i = 0; i < childSchemas.length; i++) {
                addToUnsafe(childSchemas[i], struct, i);
            }
            break;
        default:
            throw new UnsupportedOperationException("unsupported data type for unsafe storage: " + schema.getDataType());
    }
}

Example 7 with DataMapRow

use of org.apache.carbondata.core.indexstore.row.DataMapRow in project carbondata by apache.

the class BlockletDataMap method addTaskMinMaxValues.

/**
 * This method will compute min/max values at task level
 *
 * @param taskMinMaxRow
 * @param minMaxLen
 * @param carbonRowSchema
 * @param minMaxValue
 * @param ordinal
 * @param isMinValueComparison
 */
private void addTaskMinMaxValues(DataMapRow taskMinMaxRow, int[] minMaxLen, CarbonRowSchema carbonRowSchema, byte[][] minMaxValue, int ordinal, boolean isMinValueComparison) {
    DataMapRow row = taskMinMaxRow.getRow(ordinal);
    byte[][] updatedMinMaxValues = minMaxValue;
    if (null == row) {
        CarbonRowSchema[] minSchemas = ((CarbonRowSchema.StructCarbonRowSchema) carbonRowSchema).getChildSchemas();
        row = new DataMapRowImpl(minSchemas);
    } else {
        byte[][] existingMinMaxValues = getMinMaxValue(taskMinMaxRow, ordinal);
        // Compare and update min max values
        for (int i = 0; i < minMaxLen.length; i++) {
            int compare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(existingMinMaxValues[i], minMaxValue[i]);
            if (isMinValueComparison) {
                if (compare < 0) {
                    updatedMinMaxValues[i] = existingMinMaxValues[i];
                }
            } else if (compare > 0) {
                updatedMinMaxValues[i] = existingMinMaxValues[i];
            }
        }
    }
    int minMaxOrdinal = 0;
    // min/max value adding
    for (int i = 0; i < minMaxLen.length; i++) {
        row.setByteArray(updatedMinMaxValues[i], minMaxOrdinal++);
    }
    taskMinMaxRow.setRow(row, ordinal);
}

Example 8 with DataMapRow

use of org.apache.carbondata.core.indexstore.row.DataMapRow in project carbondata by apache.

the class BlockletDataMap method getFileDetails.

private String[] getFileDetails() {
    try {
        String[] fileDetails = new String[3];
        DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(0);
        fileDetails[0] = new String(unsafeRow.getByteArray(INDEX_PATH), CarbonCommonConstants.DEFAULT_CHARSET);
        fileDetails[1] = new String(unsafeRow.getByteArray(INDEX_FILE_NAME), CarbonCommonConstants.DEFAULT_CHARSET);
        fileDetails[2] = new String(unsafeRow.getByteArray(SEGMENTID), CarbonCommonConstants.DEFAULT_CHARSET);
        return fileDetails;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 9 with DataMapRow

use of org.apache.carbondata.core.indexstore.row.DataMapRow in project carbondata by apache.

the class BlockletDataMap method isScanRequired.

@Override
public boolean isScanRequired(FilterResolverIntf filterExp) {
    FilterExecuter filterExecuter = FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
    for (int i = 0; i < unsafeMemorySummaryDMStore.getRowCount(); i++) {
        DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(i);
        boolean isScanRequired = FilterExpressionProcessor.isScanRequired(filterExecuter, getMinMaxValue(unsafeRow, TASK_MAX_VALUES_INDEX), getMinMaxValue(unsafeRow, TASK_MIN_VALUES_INDEX));
        if (isScanRequired) {
            return true;
        }
    }
    return false;
}

Example 10 with DataMapRow

use of org.apache.carbondata.core.indexstore.row.DataMapRow in project carbondata by apache.

the class BlockletDataMap method loadToUnsafe.

private DataMapRowImpl loadToUnsafe(DataFileFooter fileFooter, SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow, BlockMetaInfo blockMetaInfo, int relativeBlockletId) {
    int[] minMaxLen = segmentProperties.getColumnsValueSize();
    List<BlockletInfo> blockletList = fileFooter.getBlockletList();
    CarbonRowSchema[] schema = unsafeMemoryDMStore.getSchema();
    // Add one row to maintain task level min max for segment pruning
    if (!blockletList.isEmpty() && summaryRow == null) {
        summaryRow = new DataMapRowImpl(unsafeMemorySummaryDMStore.getSchema());
    }
    for (int index = 0; index < blockletList.size(); index++) {
        DataMapRow row = new DataMapRowImpl(schema);
        int ordinal = 0;
        int taskMinMaxOrdinal = 0;
        BlockletInfo blockletInfo = blockletList.get(index);
        // add start key as index key
        row.setByteArray(blockletInfo.getBlockletIndex().getBtreeIndex().getStartKey(), ordinal++);
        BlockletMinMaxIndex minMaxIndex = blockletInfo.getBlockletIndex().getMinMaxIndex();
        byte[][] minValues = updateMinValues(minMaxIndex.getMinValues(), minMaxLen);
        row.setRow(addMinMax(minMaxLen, schema[ordinal], minValues), ordinal);
        // compute and set task level min values
        addTaskMinMaxValues(summaryRow, minMaxLen, unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], minValues, TASK_MIN_VALUES_INDEX, true);
        ordinal++;
        taskMinMaxOrdinal++;
        byte[][] maxValues = updateMaxValues(minMaxIndex.getMaxValues(), minMaxLen);
        row.setRow(addMinMax(minMaxLen, schema[ordinal], maxValues), ordinal);
        // compute and set task level max values
        addTaskMinMaxValues(summaryRow, minMaxLen, unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], maxValues, TASK_MAX_VALUES_INDEX, false);
        ordinal++;
        row.setInt(blockletInfo.getNumberOfRows(), ordinal++);
        // add file path
        byte[] filePathBytes = filePath.getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
        row.setByteArray(filePathBytes, ordinal++);
        // add pages
        row.setShort((short) blockletInfo.getNumberOfPages(), ordinal++);
        // add version number
        row.setShort(fileFooter.getVersionId().number(), ordinal++);
        // add schema updated time
        row.setLong(fileFooter.getSchemaUpdatedTimeStamp(), ordinal++);
        // add blocklet info
        byte[] serializedData;
        try {
            ByteArrayOutputStream stream = new ByteArrayOutputStream();
            DataOutput dataOutput = new DataOutputStream(stream);
            blockletInfo.write(dataOutput);
            serializedData = stream.toByteArray();
            row.setByteArray(serializedData, ordinal++);
            // Add block footer offset, it is used if we need to read footer of block
            row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);
            setLocations(blockMetaInfo.getLocationInfo(), row, ordinal);
            ordinal++;
            // for relative blockelt id i.e blocklet id that belongs to a particular part file
            row.setShort((short) relativeBlockletId++, ordinal++);
            // Store block size
            row.setLong(blockMetaInfo.getSize(), ordinal);
            unsafeMemoryDMStore.addIndexRowToUnsafe(row);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    return summaryRow;
}