Examples with CarbonDictionaryColumnMetaChunk org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk used on opensource projects

Example 6 with CarbonDictionaryColumnMetaChunk

use of org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk in project carbondata by apache.

the class CarbonDictionaryWriterImplTest method testReadingOfDictionaryChunkBetweenStartAndEndOffset.

/**
   * this method will test the reading of dictionary file between start and end offset
   */
@Test
public void testReadingOfDictionaryChunkBetweenStartAndEndOffset() throws Exception {
    // delete store path
    deleteStorePath();
    // prepare the writer to write dataset1
    CarbonDictionaryWriterImpl writer = prepareWriter();
    // write dataset1 data
    writeDictionaryFile(writer, dataSet1);
    // record dictionary file start offset
    long dictionaryStartOffset = CarbonUtil.getFileSize(this.dictionaryFilePath);
    // prepare the writer to write dataset2
    writer = prepareWriter();
    // write dataset2
    writeDictionaryFile(writer, dataSet2);
    // record the end offset for dictionary file
    long dictionaryFileEndOffset = CarbonUtil.getFileSize(this.dictionaryFilePath);
    // prepare writer to write dataset3
    writer = prepareWriter();
    // write dataset 3
    writeDictionaryFile(writer, dataSet3);
    // read dictionary chunk from dictionary file
    List<byte[]> dictionaryData = readDictionaryFile(dictionaryStartOffset, dictionaryFileEndOffset);
    // prepare the retrieved data
    List<String> actual = convertByteArrayListToStringValueList(dictionaryData);
    // compare dictionary data set
    compareDictionaryData(actual, dataSet2);
    // read chunk metadata file
    List<CarbonDictionaryColumnMetaChunk> carbonDictionaryColumnMetaChunks = readDictionaryMetadataFile();
    // assert for metadata chunk size
    assertTrue(3 == carbonDictionaryColumnMetaChunks.size());
    CarbonDictionaryColumnMetaChunk expected = new CarbonDictionaryColumnMetaChunk(3, 4, dictionaryStartOffset, dictionaryFileEndOffset, 1);
    validateDictionaryMetadata(carbonDictionaryColumnMetaChunks.get(1), expected);
}

Example 7 with CarbonDictionaryColumnMetaChunk

use of org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk in project carbondata by apache.

the class AbstractDictionaryCache method checkAndLoadDictionaryData.

/**
   * This method will get the value for the given key. If value does not exist
   * for the given key, it will check and load the value.
   *
   * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
   *                                         tableName and columnIdentifier
   * @param dictionaryInfo
   * @param lruCacheKey
   * @throws IOException                    in case memory is not sufficient to load dictionary
   *                                        into memory
   */
protected void checkAndLoadDictionaryData(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, DictionaryInfo dictionaryInfo, String lruCacheKey, boolean loadSortIndex) throws IOException {
    // read last segm
    // ent dictionary meta chunk entry to get the end offset of file
    CarbonFile carbonFile = getDictionaryMetaCarbonFile(dictionaryColumnUniqueIdentifier);
    boolean dictionaryMetaFileModified = isDictionaryMetaFileModified(carbonFile, dictionaryInfo.getFileTimeStamp(), dictionaryInfo.getDictionaryMetaFileLength());
    // meta file
    if (dictionaryMetaFileModified) {
        synchronized (dictionaryInfo) {
            carbonFile = getDictionaryMetaCarbonFile(dictionaryColumnUniqueIdentifier);
            dictionaryMetaFileModified = isDictionaryMetaFileModified(carbonFile, dictionaryInfo.getFileTimeStamp(), dictionaryInfo.getDictionaryMetaFileLength());
            // meta file
            if (dictionaryMetaFileModified) {
                CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk = readLastChunkFromDictionaryMetadataFile(dictionaryColumnUniqueIdentifier);
                // required size will be size total size of file - offset till file is
                // already read
                long requiredSize = carbonDictionaryColumnMetaChunk.getEnd_offset() - dictionaryInfo.getMemorySize();
                if (requiredSize > 0) {
                    boolean columnAddedToLRUCache = carbonLRUCache.put(lruCacheKey, dictionaryInfo, requiredSize);
                    // dictionary data
                    if (columnAddedToLRUCache) {
                        // load dictionary data
                        loadDictionaryData(dictionaryInfo, dictionaryColumnUniqueIdentifier, dictionaryInfo.getMemorySize(), carbonDictionaryColumnMetaChunk.getEnd_offset(), loadSortIndex);
                        // set the end offset till where file is read
                        dictionaryInfo.setOffsetTillFileIsRead(carbonDictionaryColumnMetaChunk.getEnd_offset());
                        dictionaryInfo.setFileTimeStamp(carbonFile.getLastModifiedTime());
                        dictionaryInfo.setDictionaryMetaFileLength(carbonFile.getSize());
                    } else {
                        throw new DictionaryBuilderException("Cannot load dictionary into memory. Not enough memory available");
                    }
                }
            }
        }
    }
    // increment the column access count
    incrementDictionaryAccessCount(dictionaryInfo);
}

Example 8 with CarbonDictionaryColumnMetaChunk

use of org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk in project carbondata by apache.

the class CarbonDictionaryWriterImpl method getChunkMetaObjectForLastSegmentEntry.

/**
   * This method will read the dictionary chunk metadata thrift object for last entry
   *
   * @return last entry of dictionary meta chunk
   * @throws IOException if an I/O error occurs
   */
private CarbonDictionaryColumnMetaChunk getChunkMetaObjectForLastSegmentEntry() throws IOException {
    CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk = null;
    CarbonDictionaryMetadataReader columnMetadataReaderImpl = getDictionaryMetadataReader();
    try {
        // read the last segment entry for dictionary metadata
        carbonDictionaryColumnMetaChunk = columnMetadataReaderImpl.readLastEntryOfDictionaryMetaChunk();
    } finally {
        // Close metadata reader
        columnMetadataReaderImpl.close();
    }
    return carbonDictionaryColumnMetaChunk;
}