Examples with ColumnDictionaryChunk org.apache.carbondata.format.ColumnDictionaryChunk used on opensource projects

Example 1 with ColumnDictionaryChunk

use of org.apache.carbondata.format.ColumnDictionaryChunk in project carbondata by apache.

the class ColumnDictionaryChunkIteratorTest method setUp.

@BeforeClass
public static void setUp() {
    ColumnDictionaryChunk columnDictionaryChunk1 = new ColumnDictionaryChunk();
    ByteBuffer byteBuffer3 = ByteBuffer.wrap("c".getBytes());
    ByteBuffer byteBuffer4 = ByteBuffer.wrap("d".getBytes());
    columnDictionaryChunk1.setValues(new ArrayList<ByteBuffer>());
    ColumnDictionaryChunk columnDictionaryChunk2 = new ColumnDictionaryChunk();
    columnDictionaryChunk2.setValues(Arrays.asList(byteBuffer3, byteBuffer4));
    expectedResult = prepareExpectedData();
    columnDictionaryChunkIterator = new ColumnDictionaryChunkIterator(Arrays.asList(columnDictionaryChunk1, columnDictionaryChunk2));
}

Example 2 with ColumnDictionaryChunk

use of org.apache.carbondata.format.ColumnDictionaryChunk in project carbondata by apache.

the class CarbonDictionaryReaderImpl method getDictionaryList.

/**
 * This method will put all the dictionary chunks into one list and return that list
 *
 * @param columnDictionaryChunks
 * @return
 */
private List<byte[]> getDictionaryList(List<ColumnDictionaryChunk> columnDictionaryChunks) {
    int dictionaryListSize = 0;
    for (ColumnDictionaryChunk dictionaryChunk : columnDictionaryChunks) {
        dictionaryListSize = dictionaryListSize + dictionaryChunk.getValues().size();
    }
    // convert byte buffer list to byte array list of dictionary values
    List<byte[]> dictionaryValues = new ArrayList<byte[]>(dictionaryListSize);
    for (ColumnDictionaryChunk dictionaryChunk : columnDictionaryChunks) {
        convertAndFillByteBufferListToByteArrayList(dictionaryValues, dictionaryChunk.getValues());
    }
    return dictionaryValues;
}

Example 3 with ColumnDictionaryChunk

use of org.apache.carbondata.format.ColumnDictionaryChunk in project carbondata by apache.

the class CarbonDictionaryWriterImpl method writeDictionaryFile.

/**
 * This method will serialize the object of dictionary file
 *
 * @throws IOException if an I/O error occurs
 */
private void writeDictionaryFile() throws IOException {
    ColumnDictionaryChunk columnDictionaryChunk = new ColumnDictionaryChunk();
    columnDictionaryChunk.setValues(oneDictionaryChunkList);
    writeThriftObject(columnDictionaryChunk);
}

Example 4 with ColumnDictionaryChunk

use of org.apache.carbondata.format.ColumnDictionaryChunk in project carbondata by apache.

the class DictionaryCacheLoaderImplTest method setUp.

@BeforeClass
public static void setUp() {
    CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier("db", "table1", "1");
    AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier.from("/tmp", carbonTableIdentifier);
    Map<String, String> columnProperties = new HashMap<>();
    columnProperties.put("prop1", "value1");
    columnProperties.put("prop2", "value2");
    columnIdentifier = new ColumnIdentifier("1", columnProperties, DataTypes.STRING);
    dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, columnIdentifier.getDataType());
    dictionaryCacheLoader = new DictionaryCacheLoaderImpl(dictionaryColumnUniqueIdentifier);
    dictionaryInfo = new ColumnDictionaryInfo(DataTypes.STRING);
    new MockUp<CarbonDictionaryReaderImpl>() {

        @Mock
        @SuppressWarnings("unused")
        Iterator<byte[]> read(long startOffset, long endOffset) throws IOException {
            ColumnDictionaryChunk columnDictionaryChunk = new ColumnDictionaryChunk();
            ByteBuffer byteBuffer1 = ByteBuffer.wrap("c".getBytes());
            ByteBuffer byteBuffer2 = ByteBuffer.wrap("d".getBytes());
            columnDictionaryChunk.setValues(Arrays.asList(byteBuffer1, byteBuffer2));
            return new ColumnDictionaryChunkIterator(Arrays.asList(columnDictionaryChunk));
        }
    };
    new MockUp<CarbonDictionarySortIndexReaderImpl>() {

        @Mock
        @SuppressWarnings("unused")
        List<Integer> readSortIndex() throws IOException {
            return Arrays.asList(1, 2);
        }

        @Mock
        @SuppressWarnings("unused")
        List<Integer> readInvertedSortIndex() throws IOException {
            return Arrays.asList(1, 2);
        }
    };
}

Example 5 with ColumnDictionaryChunk

use of org.apache.carbondata.format.ColumnDictionaryChunk in project carbondata by apache.

the class CarbonDictionaryReaderImpl method read.

/**
 * This method will be used to read data between given start and end offset.
 * Applicable scenarios:
 * 1. Truncate operation. If there is any inconsistency while writing the dictionary file
 * then we can give the start and end offset till where the data has to be retained.
 *
 * @param startOffset start offset of dictionary file
 * @param endOffset   end offset of dictionary file
 * @return iterator over byte array. Each byte array is unique dictionary value
 * @throws IOException if an I/O error occurs
 */
@Override
public Iterator<byte[]> read(long startOffset, long endOffset) throws IOException {
    List<CarbonDictionaryColumnMetaChunk> carbonDictionaryColumnMetaChunks = readDictionaryMetadataFile();
    List<ColumnDictionaryChunk> columnDictionaryChunks = read(carbonDictionaryColumnMetaChunks, startOffset, endOffset);
    return (Iterator<byte[]>) new ColumnDictionaryChunkIterator(columnDictionaryChunks);
}