Examples with KeyGenException org.apache.carbondata.core.keygenerator.KeyGenException used on opensource projects

Example 6 with KeyGenException

use of org.apache.carbondata.core.keygenerator.KeyGenException in project carbondata by apache.

the class FilterExpressionProcessor method getFilterredBlocks.

/**
 * This API will scan the Segment level all btrees and selects the required
 * block reference  nodes inorder to push the same to executer for applying filters
 * on the respective data reference node.
 * Following Algorithm is followed in below API
 * Step:1 Get the start end key based on the filter tree resolver information
 * Step:2 Prepare the IndexKeys inorder to scan the tree and get the start and end reference
 * node(block)
 * Step:3 Once data reference node ranges retrieved traverse the node within this range
 * and select the node based on the block min and max value and the filter value.
 * Step:4 The selected blocks will be send to executers for applying the filters with the help
 * of Filter executers.
 */
public List<DataRefNode> getFilterredBlocks(DataRefNode btreeNode, FilterResolverIntf filterResolver, AbstractIndex tableSegment) {
    // Need to get the current dimension tables
    List<DataRefNode> listOfDataBlocksToScan = new ArrayList<DataRefNode>();
    // selected block reference nodes based on filter resolver tree.
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("preparing the start and end key for finding" + "start and end block as per filter resolver");
    }
    IndexKey searchStartKey = null;
    IndexKey searchEndKey = null;
    try {
        searchStartKey = FilterUtil.prepareDefaultStartIndexKey(tableSegment.getSegmentProperties());
        searchEndKey = FilterUtil.prepareDefaultEndIndexKey(tableSegment.getSegmentProperties());
    } catch (KeyGenException e) {
        throw new RuntimeException(e);
    }
    if (LOGGER.isDebugEnabled()) {
        char delimiter = ',';
        LOGGER.debug("Successfully retrieved the start and end key" + "Dictionary Start Key: " + joinByteArray(searchStartKey.getDictionaryKeys(), delimiter) + "No Dictionary Start Key " + joinByteArray(searchStartKey.getNoDictionaryKeys(), delimiter) + "Dictionary End Key: " + joinByteArray(searchEndKey.getDictionaryKeys(), delimiter) + "No Dictionary End Key " + joinByteArray(searchEndKey.getNoDictionaryKeys(), delimiter));
    }
    long startTimeInMillis = System.currentTimeMillis();
    DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(tableSegment.getSegmentProperties().getEachDimColumnValueSize(), tableSegment.getSegmentProperties().getNumberOfSortColumns(), tableSegment.getSegmentProperties().getNumberOfNoDictSortColumns());
    DataRefNode startBlock = blockFinder.findFirstDataBlock(btreeNode, searchStartKey);
    DataRefNode endBlock = blockFinder.findLastDataBlock(btreeNode, searchEndKey);
    FilterExecuter filterExecuter = FilterUtil.getFilterExecuterTree(filterResolver, tableSegment.getSegmentProperties(), null);
    while (startBlock != endBlock) {
        addBlockBasedOnMinMaxValue(filterExecuter, listOfDataBlocksToScan, startBlock);
        startBlock = startBlock.getNextDataRefNode();
    }
    addBlockBasedOnMinMaxValue(filterExecuter, listOfDataBlocksToScan, endBlock);
    LOGGER.info("Total Time in retrieving the data reference node" + "after scanning the btree " + (System.currentTimeMillis() - startTimeInMillis) + " Total number of data reference node for executing filter(s) " + listOfDataBlocksToScan.size());
    return listOfDataBlocksToScan;
}

Example 7 with KeyGenException

use of org.apache.carbondata.core.keygenerator.KeyGenException in project carbondata by apache.

the class QueryUtilTest method testGetMaxKeyBasedOnDimensions.

@Test
public void testGetMaxKeyBasedOnDimensions() {
    List<ProjectionDimension> dimensions = new ArrayList<ProjectionDimension>();
    for (int i = 0; i < 2; i++) {
        ProjectionDimension dimension = new ProjectionDimension(segmentProperties.getDimensions().get(i));
        dimensions.add(dimension);
    }
    byte[] maxKeyBasedOnDimensions = null;
    try {
        maxKeyBasedOnDimensions = QueryUtil.getMaxKeyBasedOnDimensions(dimensions, segmentProperties.getDimensionKeyGenerator());
    } catch (KeyGenException e) {
        assertTrue(false);
    }
    byte[] expectedMaxKeyBasedOnDimensions = { -1, 0, 0, 0, 0, 0 };
    for (int i = 0; i < expectedMaxKeyBasedOnDimensions.length; i++) {
        if (expectedMaxKeyBasedOnDimensions[i] != maxKeyBasedOnDimensions[i]) {
            assertTrue(false);
        }
    }
    long[] expectedKeyArray = { 255, 0, 0, 0, 0, 0 };
    long[] keyArray = segmentProperties.getDimensionKeyGenerator().getKeyArray(maxKeyBasedOnDimensions);
    for (int i = 0; i < keyArray.length; i++) {
        if (expectedKeyArray[i] != keyArray[i]) {
            assertTrue(false);
        }
    }
}

Example 8 with KeyGenException

use of org.apache.carbondata.core.keygenerator.KeyGenException in project carbondata by apache.

the class ColumnGroupDimensionDataChunkTest method setup.

@BeforeClass
public static void setup() {
    int[] bitLength = CarbonUtil.getDimensionBitLength(new int[] { 10, 10, 10 }, new int[] { 3 });
    // create a key generator
    keyGenerator = new MultiDimKeyVarLengthGenerator(bitLength);
    byte[] data = new byte[keyGenerator.getKeySizeInBytes() * 3];
    int position = 0;
    for (int i = 1; i <= 3; i++) {
        try {
            System.arraycopy(keyGenerator.generateKey(new int[] { i, i, i }), 0, data, position, keyGenerator.getKeySizeInBytes());
        } catch (KeyGenException e) {
            assertTrue(false);
        }
        position += keyGenerator.getKeySizeInBytes();
    }
    columnGroupDimensionDataChunk = new ColumnGroupDimensionColumnPage(data, keyGenerator.getKeySizeInBytes(), 3);
}

Example 9 with KeyGenException

use of org.apache.carbondata.core.keygenerator.KeyGenException in project carbondata by apache.

the class TablePage method addComplexColumn.

/**
 * add a complex column into internal member compleDimensionPage
 *
 * @param index          index of the complexDimensionPage
 * @param rowId          Id of the input row
 * @param complexColumns byte array the complex columm to be added, extracted of input row
 */
// TODO: this function should be refactoried, ColumnPage should support complex type encoding
// directly instead of doing it here
private void addComplexColumn(int index, int rowId, byte[] complexColumns) {
    GenericDataType complexDataType = complexIndexMap.get(index + model.getPrimitiveDimLens().length);
    // initialize the page if first row
    if (rowId == 0) {
        int depthInComplexColumn = complexDataType.getColsCount();
        complexDimensionPages[index] = new ComplexColumnPage(pageSize, depthInComplexColumn);
    }
    int depthInComplexColumn = complexDimensionPages[index].getDepth();
    // this is the result columnar data which will be added to page,
    // size of this list is the depth of complex column, we will fill it by input data
    List<ArrayList<byte[]>> encodedComplexColumnar = new ArrayList<>(depthInComplexColumn);
    for (int k = 0; k < depthInComplexColumn; k++) {
        encodedComplexColumnar.add(new ArrayList<byte[]>());
    }
    // apply the complex type data and fill columnsArray
    try {
        ByteBuffer byteArrayInput = ByteBuffer.wrap(complexColumns);
        ByteArrayOutputStream byteArrayOutput = new ByteArrayOutputStream();
        DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutput);
        complexDataType.parseAndBitPack(byteArrayInput, dataOutputStream, model.getComplexDimensionKeyGenerator());
        complexDataType.getColumnarDataForComplexType(encodedComplexColumnar, ByteBuffer.wrap(byteArrayOutput.toByteArray()));
        byteArrayOutput.close();
    } catch (IOException | KeyGenException e) {
        throw new CarbonDataWriterException("Problem while bit packing and writing complex datatype", e);
    }
    for (int depth = 0; depth < depthInComplexColumn; depth++) {
        complexDimensionPages[index].putComplexData(rowId, depth, encodedComplexColumnar.get(depth));
    }
}

Example 10 with KeyGenException

use of org.apache.carbondata.core.keygenerator.KeyGenException in project carbondata by apache.

the class CarbonInputFormat method getDataBlocksOfIndex.

/**
   * get data blocks of given btree
   */
private List<DataRefNode> getDataBlocksOfIndex(AbstractIndex abstractIndex) {
    List<DataRefNode> blocks = new LinkedList<DataRefNode>();
    SegmentProperties segmentProperties = abstractIndex.getSegmentProperties();
    try {
        IndexKey startIndexKey = FilterUtil.prepareDefaultStartIndexKey(segmentProperties);
        IndexKey endIndexKey = FilterUtil.prepareDefaultEndIndexKey(segmentProperties);
        // Add all blocks of btree into result
        DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(), segmentProperties.getNumberOfSortColumns(), segmentProperties.getNumberOfNoDictSortColumns());
        DataRefNode startBlock = blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
        DataRefNode endBlock = blockFinder.findLastDataBlock(abstractIndex.getDataRefNode(), endIndexKey);
        while (startBlock != endBlock) {
            blocks.add(startBlock);
            startBlock = startBlock.getNextDataRefNode();
        }
        blocks.add(endBlock);
    } catch (KeyGenException e) {
        LOG.error("Could not generate start key", e);
    }
    return blocks;
}