Examples with SegmentProperties org.apache.carbondata.core.datastore.block.SegmentProperties used on opensource projects

Example 6 with SegmentProperties

use of org.apache.carbondata.core.datastore.block.SegmentProperties in project carbondata by apache.

the class RestructureBasedRawResultCollector method initCurrentBlockKeyGenerator.

/**
   * This method will initialize the block key generator for the current block based on the
   * dictionary columns present in the current block
   */
private void initCurrentBlockKeyGenerator() {
    SegmentProperties segmentProperties = tableBlockExecutionInfos.getDataBlock().getSegmentProperties();
    int[] dictionaryColumnBlockIndex = tableBlockExecutionInfos.getDictionaryColumnBlockIndex();
    int[] updatedColumnCardinality = new int[dictionaryColumnBlockIndex.length];
    int[] updatedDimensionPartitioner = new int[dictionaryColumnBlockIndex.length];
    for (int i = 0; i < dictionaryColumnBlockIndex.length; i++) {
        // get the dictionary key ordinal as column cardinality in segment properties
        // will only be for dictionary encoded columns
        CarbonDimension currentBlockDimension = segmentProperties.getDimensions().get(dictionaryColumnBlockIndex[i]);
        updatedColumnCardinality[i] = segmentProperties.getDimColumnsCardinality()[currentBlockDimension.getKeyOrdinal()];
        updatedDimensionPartitioner[i] = segmentProperties.getDimensionPartitions()[currentBlockDimension.getKeyOrdinal()];
    }
    if (dictionaryColumnBlockIndex.length > 0) {
        int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(updatedColumnCardinality, updatedDimensionPartitioner);
        updatedCurrentBlockKeyGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
    }
}

Example 7 with SegmentProperties

use of org.apache.carbondata.core.datastore.block.SegmentProperties in project carbondata by apache.

the class CarbonFooterWriterTest method testWriteFactMetadata.

/**
   * test writing fact metadata.
   */
@Test
public void testWriteFactMetadata() throws IOException {
    deleteFile();
    createFile();
    CarbonFooterWriter writer = new CarbonFooterWriter(filePath);
    List<BlockletInfoColumnar> infoColumnars = getBlockletInfoColumnars();
    int[] cardinalities = new int[] { 2, 4, 5, 7, 9, 10 };
    List<ColumnSchema> columnSchema = Arrays.asList(new ColumnSchema[] { getDimensionColumn("IMEI1"), getDimensionColumn("IMEI2"), getDimensionColumn("IMEI3"), getDimensionColumn("IMEI4"), getDimensionColumn("IMEI5"), getDimensionColumn("IMEI6") });
    List<org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema> wrapperColumnSchema = Arrays.asList(new org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema[] { getWrapperDimensionColumn("IMEI1"), getWrapperDimensionColumn("IMEI2"), getWrapperDimensionColumn("IMEI3"), getWrapperDimensionColumn("IMEI4"), getWrapperDimensionColumn("IMEI5"), getWrapperDimensionColumn("IMEI6") });
    int[] colCardinality = CarbonUtil.getFormattedCardinality(cardinalities, wrapperColumnSchema);
    SegmentProperties segmentProperties = new SegmentProperties(wrapperColumnSchema, colCardinality);
    writer.writeFooter(CarbonMetadataUtil.convertFileFooter(infoColumnars, 6, cardinalities, columnSchema, segmentProperties), 0);
    CarbonFooterReader metaDataReader = new CarbonFooterReader(filePath, 0);
    assertTrue(metaDataReader.readFooter() != null);
}

Example 8 with SegmentProperties

use of org.apache.carbondata.core.datastore.block.SegmentProperties in project carbondata by apache.

the class CarbonCompactionExecutor method getSourceSegmentProperties.

/**
   * This method will create the source segment properties based on restructured block existence
   *
   * @param listMetadata
   * @return
   */
private SegmentProperties getSourceSegmentProperties(List<DataFileFooter> listMetadata) {
    SegmentProperties sourceSegProperties = null;
    if (restructuredBlockExists) {
        // update cardinality of source segment according to new schema
        Map<String, Integer> columnToCardinalityMap = new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
        CarbonCompactionUtil.addColumnCardinalityToMap(columnToCardinalityMap, listMetadata.get(0).getColumnInTable(), listMetadata.get(0).getSegmentInfo().getColumnCardinality());
        List<ColumnSchema> updatedColumnSchemaList = new ArrayList<>(listMetadata.get(0).getColumnInTable().size());
        int[] updatedColumnCardinalities = CarbonCompactionUtil.updateColumnSchemaAndGetCardinality(columnToCardinalityMap, carbonTable, updatedColumnSchemaList);
        sourceSegProperties = new SegmentProperties(updatedColumnSchemaList, updatedColumnCardinalities);
    } else {
        sourceSegProperties = new SegmentProperties(listMetadata.get(0).getColumnInTable(), listMetadata.get(0).getSegmentInfo().getColumnCardinality());
    }
    return sourceSegProperties;
}

Example 9 with SegmentProperties

use of org.apache.carbondata.core.datastore.block.SegmentProperties in project carbondata by apache.

the class CarbonCompactionExecutor method processTableBlocks.

/**
   * For processing of the table blocks.
   *
   * @return List of Carbon iterators
   */
public List<RawResultIterator> processTableBlocks() throws QueryExecutionException, IOException {
    List<RawResultIterator> resultList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    List<TableBlockInfo> list = null;
    queryModel = prepareQueryModel(list);
    // iterate each seg ID
    for (Map.Entry<String, TaskBlockInfo> taskMap : segmentMapping.entrySet()) {
        String segmentId = taskMap.getKey();
        List<DataFileFooter> listMetadata = dataFileMetadataSegMapping.get(segmentId);
        SegmentProperties sourceSegProperties = getSourceSegmentProperties(listMetadata);
        // for each segment get taskblock info
        TaskBlockInfo taskBlockInfo = taskMap.getValue();
        Set<String> taskBlockListMapping = taskBlockInfo.getTaskSet();
        for (String task : taskBlockListMapping) {
            list = taskBlockInfo.getTableBlockInfoList(task);
            Collections.sort(list);
            LOGGER.info("for task -" + task + "-block size is -" + list.size());
            queryModel.setTableBlockInfos(list);
            resultList.add(new RawResultIterator(executeBlockList(list), sourceSegProperties, destinationSegProperties));
        }
    }
    return resultList;
}

Example 10 with SegmentProperties

use of org.apache.carbondata.core.datastore.block.SegmentProperties in project carbondata by apache.

the class CarbonMetadataUtilTest method testConvertFileFooter.

@Test
public void testConvertFileFooter() throws Exception {
    int[] intArr = { 1, 2, 3, 4, 5 };
    boolean[] boolArr = { true, true, true, true, true };
    long[] longArr = { 1, 2, 3, 4, 5 };
    byte[][] maxByteArr = { { 1, 2 }, { 3, 4 }, { 5, 6 }, { 2, 4 }, { 1, 2 } };
    int[] cardinality = { 1, 2, 3, 4, 5 };
    org.apache.carbondata.core.metadata.datatype.DataType[] dataType = { org.apache.carbondata.core.metadata.datatype.DataType.INT, org.apache.carbondata.core.metadata.datatype.DataType.INT, org.apache.carbondata.core.metadata.datatype.DataType.INT, org.apache.carbondata.core.metadata.datatype.DataType.INT, org.apache.carbondata.core.metadata.datatype.DataType.INT };
    org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema colSchema = new org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema();
    org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema colSchema1 = new org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema();
    List<org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema> columnSchemaList = new ArrayList<>();
    columnSchemaList.add(colSchema);
    columnSchemaList.add(colSchema1);
    SegmentProperties segmentProperties = new SegmentProperties(columnSchemaList, cardinality);
    final List<Integer> integerList = new ArrayList<>();
    integerList.add(new Integer("1"));
    integerList.add(new Integer("2"));
    WriterCompressModel writerCompressModel = new WriterCompressModel();
    writerCompressModel.setMaxValue(objMaxArr);
    writerCompressModel.setMinValue(objMinArr);
    writerCompressModel.setDataTypeSelected(byteArr);
    writerCompressModel.setMantissa(intArr);
    writerCompressModel.setType(dataType);
    writerCompressModel.setUniqueValue(objMinArr);
    BlockletInfoColumnar blockletInfoColumnar = new BlockletInfoColumnar();
    BitSet[] bitSetArr = new BitSet[6];
    bitSetArr[0] = new BitSet();
    bitSetArr[1] = new BitSet();
    bitSetArr[2] = new BitSet();
    bitSetArr[3] = new BitSet();
    bitSetArr[4] = new BitSet();
    bitSetArr[5] = new BitSet();
    blockletInfoColumnar.setColumnMaxData(maxByteArr);
    blockletInfoColumnar.setColumnMinData(maxByteArr);
    blockletInfoColumnar.setKeyLengths(intArr);
    blockletInfoColumnar.setColGrpBlocks(boolArr);
    blockletInfoColumnar.setKeyOffSets(longArr);
    blockletInfoColumnar.setDataIndexMapOffsets(longArr);
    blockletInfoColumnar.setAggKeyBlock(boolArr);
    blockletInfoColumnar.setDataIndexMapLength(intArr);
    blockletInfoColumnar.setIsSortedKeyColumn(boolArr);
    blockletInfoColumnar.setKeyOffSets(longArr);
    blockletInfoColumnar.setMeasureLength(intArr);
    blockletInfoColumnar.setMeasureOffset(longArr);
    blockletInfoColumnar.setMeasureNullValueIndex(bitSetArr);
    blockletInfoColumnar.setCompressionModel(writerCompressModel);
    BlockletInfoColumnar blockletInfoColumnar1 = new BlockletInfoColumnar();
    blockletInfoColumnar1.setColumnMaxData(maxByteArr);
    blockletInfoColumnar1.setColumnMinData(maxByteArr);
    blockletInfoColumnar1.setKeyLengths(intArr);
    blockletInfoColumnar1.setKeyOffSets(longArr);
    blockletInfoColumnar1.setDataIndexMapOffsets(longArr);
    blockletInfoColumnar1.setAggKeyBlock(boolArr);
    blockletInfoColumnar1.setDataIndexMapLength(intArr);
    blockletInfoColumnar1.setIsSortedKeyColumn(boolArr);
    blockletInfoColumnar1.setColGrpBlocks(boolArr);
    blockletInfoColumnar1.setKeyOffSets(longArr);
    blockletInfoColumnar1.setMeasureLength(intArr);
    blockletInfoColumnar1.setMeasureOffset(longArr);
    blockletInfoColumnar1.setMeasureNullValueIndex(bitSetArr);
    blockletInfoColumnar1.setCompressionModel(writerCompressModel);
    blockletInfoColumnar1.setColGrpBlocks(boolArr);
    List<BlockletInfoColumnar> blockletInfoColumnarList = new ArrayList<>();
    blockletInfoColumnarList.add(blockletInfoColumnar);
    blockletInfoColumnarList.add(blockletInfoColumnar1);
    new MockUp<CarbonUtil>() {

        @SuppressWarnings("unused")
        @Mock
        public List<Integer> convertToIntegerList(int[] array) {
            return integerList;
        }
    };
    final Set<Integer> integerSet = new HashSet<>();
    integerSet.add(new Integer("1"));
    integerSet.add(new Integer("2"));
    new MockUp<SegmentProperties>() {

        @SuppressWarnings("unused")
        @Mock
        public Set<Integer> getDimensionOrdinalForBlock(int blockIndex) {
            return integerSet;
        }
    };
    SegmentInfo segmentInfo = new SegmentInfo();
    segmentInfo.setNum_cols(4);
    segmentInfo.setColumn_cardinalities(integerList);
    FileFooter fileFooter = new FileFooter();
    fileFooter.setNum_rows(4);
    fileFooter.setSegment_info(segmentInfo);
    byte[] byteMaxArr = "1".getBytes();
    byte[] byteMinArr = "2".getBytes();
    BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();
    blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(byteMaxArr));
    blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(byteMinArr));
    FileFooter result = convertFileFooter(blockletInfoColumnarList, 4, cardinality, columnSchemas, segmentProperties);
    assertEquals(result.getTable_columns(), columnSchemas);
}