Examples with FixedLengthDimensionColumnPage org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage used on opensource projects

Example 16 with FixedLengthDimensionColumnPage

use of org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage in project carbondata by apache.

the class CarbonUtilTest method IndexUsingBinarySearchLengthThree.

@Test
public void IndexUsingBinarySearchLengthThree() {
    byte[] dataChunk = new byte[10];
    FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk;
    byte[] keyWord = new byte[3];
    dataChunk = "aaabbbbbbbbbccc".getBytes();
    byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length];
    fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null, dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
    for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) {
        dataArr[ii] = fixedLengthDimensionDataChunk.getChunkData(ii);
    }
    keyWord[0] = Byte.valueOf("98");
    keyWord[1] = Byte.valueOf("98");
    keyWord[2] = Byte.valueOf("98");
    int[] expectRangeIndex = new int[2];
    expectRangeIndex[0] = 1;
    expectRangeIndex[1] = 3;
    assertRangeIndex(dataArr, dataChunk, fixedLengthDimensionDataChunk, keyWord, expectRangeIndex);
}

Example 17 with FixedLengthDimensionColumnPage

use of org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage in project carbondata by apache.

the class CarbonUtilTest method testToGetnextGreaterValue.

@Test
public void testToGetnextGreaterValue() {
    byte[] dataChunks = { 5, 6, 7, 8, 9 };
    byte[] compareValues = { 7 };
    FixedLengthDimensionColumnPage fixedLengthDataChunk = new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1, dataChunks.length);
    int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk, compareValues, 5);
    assertEquals(result, 3);
}

Example 18 with FixedLengthDimensionColumnPage

use of org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage in project carbondata by apache.

the class CarbonUtilTest method IndexUsingBinarySearchLengthTwo.

@Test
public void IndexUsingBinarySearchLengthTwo() {
    byte[] dataChunk = new byte[10];
    FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk;
    byte[] keyWord = new byte[2];
    dataChunk = "aabbbbbbbbbbcc".getBytes();
    byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length];
    fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null, dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
    for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) {
        dataArr[ii] = fixedLengthDimensionDataChunk.getChunkData(ii);
    }
    keyWord[0] = Byte.valueOf("98");
    keyWord[1] = Byte.valueOf("98");
    int[] expectRangeIndex = new int[2];
    expectRangeIndex[0] = 1;
    expectRangeIndex[1] = 5;
    assertRangeIndex(dataArr, dataChunk, fixedLengthDimensionDataChunk, keyWord, expectRangeIndex);
    keyWord[0] = Byte.valueOf("97");
    keyWord[1] = Byte.valueOf("97");
    expectRangeIndex = new int[2];
    expectRangeIndex[0] = 0;
    expectRangeIndex[1] = 0;
    assertRangeIndex(dataArr, dataChunk, fixedLengthDimensionDataChunk, keyWord, expectRangeIndex);
    keyWord[0] = Byte.valueOf("99");
    keyWord[1] = Byte.valueOf("99");
    expectRangeIndex = new int[2];
    expectRangeIndex[0] = 6;
    expectRangeIndex[1] = 6;
    assertRangeIndex(dataArr, dataChunk, fixedLengthDimensionDataChunk, keyWord, expectRangeIndex);
}

Example 19 with FixedLengthDimensionColumnPage

use of org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage in project carbondata by apache.

the class IncludeFilterExecutorImplTest method comparePerformance.

/**
 * comapre result and performance
 *
 * @param dataChunkSize dataChunk's stored data size
 * @param filteredValueCnt filtered dictionary value count
 * @param queryTimes repeat query times in the test
 * @param repeatTimes repeated times for a dictionary value
 * @return
 */
private void comparePerformance(int dataChunkSize, int filteredValueCnt, int queryTimes, int repeatTimes) {
    long start;
    long oldTime = 0;
    long newTime = 0;
    // used to generate filter value
    int baseFilteredValue = 100;
    // column dictionary size
    int dimColumnSize = 2;
    if ((dataChunkSize / repeatTimes) <= 255 && (baseFilteredValue + filteredValueCnt) <= 255) {
        dimColumnSize = 1;
    }
    System.out.println("dimColumnSize: " + dimColumnSize);
    FixedLengthDimensionColumnPage dimensionColumnDataChunk;
    DimColumnExecutorFilterInfo dim = new DimColumnExecutorFilterInfo();
    byte[] dataChunk = new byte[dataChunkSize * dimColumnSize];
    for (int i = 0; i < dataChunkSize; i++) {
        if (i % repeatTimes == 0) {
            repeatTimes++;
        }
        byte[] data = transferIntToByteArr(repeatTimes, dimColumnSize);
        for (int j = 0; j < dimColumnSize; j++) {
            dataChunk[dimColumnSize * i + j] = data[j];
        }
    }
    byte[][] filterKeys = new byte[filteredValueCnt][2];
    for (int k = 0; k < filteredValueCnt; k++) {
        filterKeys[k] = transferIntToByteArr(baseFilteredValue + k, dimColumnSize);
    }
    dim.setFilterKeys(filterKeys);
    dimensionColumnDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null, dataChunkSize, dimColumnSize, dataChunk.length);
    // repeat query and compare 2 result between old code and new optimized code
    for (int j = 0; j < queryTimes; j++) {
        start = System.currentTimeMillis();
        BitSet bitOld = this.setFilterdIndexToBitSet(dimensionColumnDataChunk, dataChunkSize, filterKeys);
        oldTime = oldTime + System.currentTimeMillis() - start;
        start = System.currentTimeMillis();
        BitSet bitNew = this.setFilterdIndexToBitSetNew(dimensionColumnDataChunk, dataChunkSize, filterKeys);
        newTime = newTime + System.currentTimeMillis() - start;
        assertTrue(bitOld.equals(bitNew));
    }
    if (filteredValueCnt >= 100) {
        Assert.assertTrue(newTime <= oldTime);
    }
    System.out.println("old code performance time: " + oldTime + " ms");
    System.out.println("new code performance time: " + newTime + " ms");
    System.out.println("filteredValueCnt: " + filteredValueCnt);
}