Examples with CarbonDictionaryColumnMetaChunk org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk used on opensource projects

Example 1 with CarbonDictionaryColumnMetaChunk

use of org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk in project carbondata by apache.

the class CarbonDictionarySortIndexReaderImpl method initPath.

protected void initPath() {
    PathService pathService = CarbonCommonFactory.getPathService();
    CarbonTablePath carbonTablePath = pathService.getCarbonTablePath(carbonStorePath, carbonTableIdentifier);
    try {
        CarbonDictionaryColumnMetaChunk chunkMetaObjectForLastSegmentEntry = getChunkMetaObjectForLastSegmentEntry();
        long dictOffset = chunkMetaObjectForLastSegmentEntry.getEnd_offset();
        this.sortIndexFilePath = carbonTablePath.getSortIndexFilePath(columnIdentifier.getColumnId(), dictOffset);
        if (!FileFactory.isFileExist(this.sortIndexFilePath, FileFactory.getFileType(this.sortIndexFilePath))) {
            this.sortIndexFilePath = carbonTablePath.getSortIndexFilePath(columnIdentifier.getColumnId());
        }
    } catch (IOException e) {
        this.sortIndexFilePath = carbonTablePath.getSortIndexFilePath(columnIdentifier.getColumnId());
    }
}

Example 2 with CarbonDictionaryColumnMetaChunk

use of org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk in project carbondata by apache.

the class AbstractDictionaryCache method readLastChunkFromDictionaryMetadataFile.

/**
   * This method will read dictionary metadata file and return the dictionary meta chunks
   *
   * @param dictionaryColumnUniqueIdentifier
   * @return list of dictionary metadata chunks
   * @throws IOException read and close method throws IO exception
   */
protected CarbonDictionaryColumnMetaChunk readLastChunkFromDictionaryMetadataFile(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException {
    DictionaryService dictService = CarbonCommonFactory.getDictionaryService();
    CarbonDictionaryMetadataReader columnMetadataReaderImpl = dictService.getDictionaryMetadataReader(dictionaryColumnUniqueIdentifier.getCarbonTableIdentifier(), dictionaryColumnUniqueIdentifier.getColumnIdentifier(), carbonStorePath);
    CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk = null;
    // read metadata file
    try {
        carbonDictionaryColumnMetaChunk = columnMetadataReaderImpl.readLastEntryOfDictionaryMetaChunk();
    } finally {
        // close the metadata reader
        columnMetadataReaderImpl.close();
    }
    return carbonDictionaryColumnMetaChunk;
}

Example 3 with CarbonDictionaryColumnMetaChunk

use of org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk in project carbondata by apache.

the class CarbonDictionaryWriterImplTest method testWriteMultipleChunksForOneSegment.

/**
   * test writing multiple dictionary chunks for a single segment
   */
@Test
public void testWriteMultipleChunksForOneSegment() throws IOException {
    deleteStorePath();
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.DICTIONARY_ONE_CHUNK_SIZE, "1");
    // prepare dictionary writer object
    CarbonDictionaryWriterImpl writer = prepareWriter();
    writeDictionaryFile(writer, dataSet1);
    // record file size from where data has to be read
    long end_offset = CarbonUtil.getFileSize(this.dictionaryFilePath);
    // read metadata chunks from file
    List<CarbonDictionaryColumnMetaChunk> carbonDictionaryColumnMetaChunks = readDictionaryMetadataFile();
    assertTrue(1 == carbonDictionaryColumnMetaChunks.size());
    // prepare retrieved chunk metadata
    long start_offset = 0L;
    CarbonDictionaryColumnMetaChunk expected = new CarbonDictionaryColumnMetaChunk(1, dataSet1.size(), start_offset, end_offset, dataSet1.size());
    // validate chunk metadata - actual and expected
    for (CarbonDictionaryColumnMetaChunk chunk : carbonDictionaryColumnMetaChunks) {
        validateDictionaryMetadata(chunk, expected);
    }
    //assert for chunk count
    List<byte[]> dictionaryValues = readDictionaryFile(0L, 0L);
    // prepare expected dictionary chunk list
    List<String> actual = convertByteArrayListToStringValueList(dictionaryValues);
    assertTrue(dataSet1.size() == actual.size());
    // validate the dictionary data
    compareDictionaryData(actual, dataSet1);
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.DICTIONARY_ONE_CHUNK_SIZE, CarbonCommonConstants.DICTIONARY_ONE_CHUNK_SIZE_DEFAULT);
}

Example 4 with CarbonDictionaryColumnMetaChunk

use of org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk in project carbondata by apache.

the class CarbonDictionaryWriterImplTest method processColumnValuesForOneChunk.

/**
   * this method will test the functionality of writing and reading one dictionary chunk
   */
private void processColumnValuesForOneChunk(int chunkCountForSegment) throws IOException {
    // delete store path
    deleteStorePath();
    // prepare writer
    CarbonDictionaryWriterImpl writer = prepareWriter();
    // write the data into file
    // test write api for passing list of byte array
    writer.write(convertStringListToByteArray(dataSet1));
    // close the writer
    writer.close();
    //write metadata
    writer.commit();
    // record end offset of file
    long end_offset = CarbonUtil.getFileSize(this.dictionaryFilePath);
    // read dictionary chunk from dictionary file
    List<byte[]> dictionaryData = readDictionaryFile(0L, 0L);
    // prepare the retrieved data
    List<String> actual = convertByteArrayListToStringValueList(dictionaryData);
    // compare the expected and actual data
    compareDictionaryData(actual, dataSet1);
    // read dictionary metadata chunks
    List<CarbonDictionaryColumnMetaChunk> carbonDictionaryColumnMetaChunks = readDictionaryMetadataFile();
    // assert
    assertTrue(1 == carbonDictionaryColumnMetaChunks.size());
    long start_offset = 0L;
    // validate actual chunk metadata with expected
    CarbonDictionaryColumnMetaChunk expected = new CarbonDictionaryColumnMetaChunk(1, 2, start_offset, end_offset, 1);
    for (CarbonDictionaryColumnMetaChunk chunk : carbonDictionaryColumnMetaChunks) {
        validateDictionaryMetadata(chunk, expected);
    }
}

Example 5 with CarbonDictionaryColumnMetaChunk

use of org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk in project carbondata by apache.

the class CarbonDictionaryWriterImplTest method readDictionaryMetadataFile.

/**
   * This method will read dictionary metadata file and return the dictionary meta chunks
   *
   * @return list of dictionary metadata chunks
   * @throws IOException read and close method throws IO excpetion
   */
private List<CarbonDictionaryColumnMetaChunk> readDictionaryMetadataFile() throws IOException {
    CarbonDictionaryMetadataReaderImpl columnMetadataReaderImpl = new CarbonDictionaryMetadataReaderImpl(this.carbonStorePath, this.carbonTableIdentifier, this.columnIdentifier);
    List<CarbonDictionaryColumnMetaChunk> dictionaryMetaChunkList = null;
    // read metadata file
    try {
        dictionaryMetaChunkList = columnMetadataReaderImpl.read();
    } finally {
        // close the metadata reader
        columnMetadataReaderImpl.close();
    }
    return dictionaryMetaChunkList;
}