Examples with SegmentProperties org.apache.carbondata.core.datastore.block.SegmentProperties used on opensource projects

Example 11 with SegmentProperties

use of org.apache.carbondata.core.datastore.block.SegmentProperties in project carbondata by apache.

the class FilterUtilTest method testPrepareDefaultStartIndexKey.

@Test
public void testPrepareDefaultStartIndexKey() throws KeyGenException {
    List<ColumnSchema> columnsInTable = new ArrayList<>();
    columnsInTable.add(columnSchema);
    int[] columnCardinality = new int[] { 1, 2 };
    new MockUp<ColumnSchema>() {

        @Mock
        public List<Encoding> getEncodingList() {
            List<Encoding> encodingList = new ArrayList<>();
            encodingList.add(Encoding.DICTIONARY);
            return encodingList;
        }
    };
    SegmentProperties segmentProperties = new SegmentProperties(columnsInTable, columnCardinality);
    assertTrue(FilterUtil.prepareDefaultStartIndexKey(segmentProperties) instanceof IndexKey);
}

Example 12 with SegmentProperties

use of org.apache.carbondata.core.datastore.block.SegmentProperties in project carbondata by apache.

the class RestructureBasedRawResultCollector method initRestructuredKeyGenerator.

/**
   * This method will create a new key generator for generating mdKey according to latest schema
   */
private void initRestructuredKeyGenerator() {
    SegmentProperties segmentProperties = tableBlockExecutionInfos.getDataBlock().getSegmentProperties();
    QueryDimension[] queryDimensions = tableBlockExecutionInfos.getActualQueryDimensions();
    List<Integer> updatedColumnCardinality = new ArrayList<>(queryDimensions.length);
    List<Integer> updatedDimensionPartitioner = new ArrayList<>(queryDimensions.length);
    int[] dictionaryColumnBlockIndex = tableBlockExecutionInfos.getDictionaryColumnBlockIndex();
    int dimCounterInCurrentBlock = 0;
    for (int i = 0; i < queryDimensions.length; i++) {
        if (queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) {
            if (tableBlockExecutionInfos.getDimensionInfo().getDimensionExists()[i]) {
                // get the dictionary key ordinal as column cardinality in segment properties
                // will only be for dictionary encoded columns
                CarbonDimension currentBlockDimension = segmentProperties.getDimensions().get(dictionaryColumnBlockIndex[dimCounterInCurrentBlock]);
                updatedColumnCardinality.add(segmentProperties.getDimColumnsCardinality()[currentBlockDimension.getKeyOrdinal()]);
                updatedDimensionPartitioner.add(segmentProperties.getDimensionPartitions()[currentBlockDimension.getKeyOrdinal()]);
                dimCounterInCurrentBlock++;
            } else {
                // partitioner index will be 1 every column will be in columnar format
                updatedDimensionPartitioner.add(1);
                // for direct dictionary 4 bytes need to be allocated else 1
                if (queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
                    updatedColumnCardinality.add(Integer.MAX_VALUE);
                } else {
                    // cardinality will be 2 will user has provided a default value
                    byte[] defaultValue = queryDimensions[i].getDimension().getDefaultValue();
                    if (null != defaultValue) {
                        updatedColumnCardinality.add(CarbonCommonConstants.DICTIONARY_DEFAULT_CARDINALITY + 1);
                    } else {
                        updatedColumnCardinality.add(CarbonCommonConstants.DICTIONARY_DEFAULT_CARDINALITY);
                    }
                }
            }
        }
    }
    if (!updatedColumnCardinality.isEmpty()) {
        int[] latestColumnCardinality = ArrayUtils.toPrimitive(updatedColumnCardinality.toArray(new Integer[updatedColumnCardinality.size()]));
        int[] latestColumnPartitioner = ArrayUtils.toPrimitive(updatedDimensionPartitioner.toArray(new Integer[updatedDimensionPartitioner.size()]));
        int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(latestColumnCardinality, latestColumnPartitioner);
        restructuredKeyGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
    }
}

Example 13 with SegmentProperties

use of org.apache.carbondata.core.datastore.block.SegmentProperties in project carbondata by apache.

the class RestructureBasedRawResultCollector method initCurrentBlockKeyGenerator.

/**
   * This method will initialize the block key generator for the current block based on the
   * dictionary columns present in the current block
   */
private void initCurrentBlockKeyGenerator() {
    SegmentProperties segmentProperties = tableBlockExecutionInfos.getDataBlock().getSegmentProperties();
    int[] dictionaryColumnBlockIndex = tableBlockExecutionInfos.getDictionaryColumnBlockIndex();
    int[] updatedColumnCardinality = new int[dictionaryColumnBlockIndex.length];
    int[] updatedDimensionPartitioner = new int[dictionaryColumnBlockIndex.length];
    for (int i = 0; i < dictionaryColumnBlockIndex.length; i++) {
        // get the dictionary key ordinal as column cardinality in segment properties
        // will only be for dictionary encoded columns
        CarbonDimension currentBlockDimension = segmentProperties.getDimensions().get(dictionaryColumnBlockIndex[i]);
        updatedColumnCardinality[i] = segmentProperties.getDimColumnsCardinality()[currentBlockDimension.getKeyOrdinal()];
        updatedDimensionPartitioner[i] = segmentProperties.getDimensionPartitions()[currentBlockDimension.getKeyOrdinal()];
    }
    if (dictionaryColumnBlockIndex.length > 0) {
        int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(updatedColumnCardinality, updatedDimensionPartitioner);
        updatedCurrentBlockKeyGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
    }
}

Example 14 with SegmentProperties

use of org.apache.carbondata.core.datastore.block.SegmentProperties in project carbondata by apache.

the class CarbonFooterWriterTest method testWriteFactMetadata.

/**
   * test writing fact metadata.
   */
@Test
public void testWriteFactMetadata() throws IOException {
    deleteFile();
    createFile();
    CarbonFooterWriter writer = new CarbonFooterWriter(filePath);
    List<BlockletInfoColumnar> infoColumnars = getBlockletInfoColumnars();
    int[] cardinalities = new int[] { 2, 4, 5, 7, 9, 10 };
    List<ColumnSchema> columnSchema = Arrays.asList(new ColumnSchema[] { getDimensionColumn("IMEI1"), getDimensionColumn("IMEI2"), getDimensionColumn("IMEI3"), getDimensionColumn("IMEI4"), getDimensionColumn("IMEI5"), getDimensionColumn("IMEI6") });
    List<org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema> wrapperColumnSchema = Arrays.asList(new org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema[] { getWrapperDimensionColumn("IMEI1"), getWrapperDimensionColumn("IMEI2"), getWrapperDimensionColumn("IMEI3"), getWrapperDimensionColumn("IMEI4"), getWrapperDimensionColumn("IMEI5"), getWrapperDimensionColumn("IMEI6") });
    int[] colCardinality = CarbonUtil.getFormattedCardinality(cardinalities, wrapperColumnSchema);
    SegmentProperties segmentProperties = new SegmentProperties(wrapperColumnSchema, colCardinality);
    writer.writeFooter(CarbonMetadataUtil.convertFileFooter(infoColumnars, 6, cardinalities, columnSchema, segmentProperties), 0);
    CarbonFooterReader metaDataReader = new CarbonFooterReader(filePath, 0);
    assertTrue(metaDataReader.readFooter() != null);
}