Examples with CarbonDictionaryWriterImpl org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl used on opensource projects

Example 1 with CarbonDictionaryWriterImpl

use of org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl in project carbondata by apache.

the class AbstractDictionaryCacheTest method prepareWriterAndWriteData.

/**
   * write dictionary data
   *
   * @param data
   * @throws IOException
   */
protected void prepareWriterAndWriteData(List<String> data, String columnId) throws IOException {
    ColumnIdentifier columnIdentifier = new ColumnIdentifier(columnId, null, null);
    CarbonDictionaryWriter carbonDictionaryWriter = new CarbonDictionaryWriterImpl(carbonStorePath, carbonTableIdentifier, columnIdentifier);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonStorePath, carbonTableIdentifier);
    CarbonUtil.checkAndCreateFolder(carbonTablePath.getMetadataDirectoryPath());
    List<byte[]> valueList = convertStringListToByteArray(data);
    try {
        carbonDictionaryWriter.write(valueList);
    } finally {
        carbonDictionaryWriter.close();
        carbonDictionaryWriter.commit();
    }
}

Example 2 with CarbonDictionaryWriterImpl

use of org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl in project carbondata by apache.

the class CarbonDictionarySortIndexReaderImplTest method read.

/**
   * Test to read the data from dictionary sort index file
   *
   * @throws Exception
   */
@Test
public void read() throws Exception {
    deleteStorePath();
    CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier("testSchema", "carbon", UUID.randomUUID().toString());
    ColumnIdentifier columnIdentifier = new ColumnIdentifier("Name", null, null);
    CarbonDictionaryWriter dictionaryWriter = new CarbonDictionaryWriterImpl(storePath, carbonTableIdentifier, columnIdentifier);
    String metaFolderPath = storePath + File.separator + carbonTableIdentifier.getDatabaseName() + File.separator + carbonTableIdentifier.getTableName() + File.separator + "Metadata";
    CarbonUtil.checkAndCreateFolder(metaFolderPath);
    CarbonDictionarySortIndexWriter dictionarySortIndexWriter = new CarbonDictionarySortIndexWriterImpl(carbonTableIdentifier, columnIdentifier, storePath);
    List<int[]> expectedData = prepareExpectedData();
    int[] data = expectedData.get(0);
    for (int i = 0; i < data.length; i++) {
        dictionaryWriter.write(String.valueOf(data[i]));
    }
    dictionaryWriter.close();
    dictionaryWriter.commit();
    List<Integer> sortIndex = Arrays.asList(ArrayUtils.toObject(expectedData.get(0)));
    List<Integer> invertedSortIndex = Arrays.asList(ArrayUtils.toObject(expectedData.get(1)));
    dictionarySortIndexWriter.writeSortIndex(sortIndex);
    dictionarySortIndexWriter.writeInvertedSortIndex(invertedSortIndex);
    dictionarySortIndexWriter.close();
    CarbonDictionarySortIndexReader dictionarySortIndexReader = new CarbonDictionarySortIndexReaderImpl(carbonTableIdentifier, columnIdentifier, storePath);
    List<Integer> actualSortIndex = dictionarySortIndexReader.readSortIndex();
    List<Integer> actualInvertedSortIndex = dictionarySortIndexReader.readInvertedSortIndex();
    for (int i = 0; i < actualSortIndex.size(); i++) {
        Assert.assertEquals(sortIndex.get(i), actualSortIndex.get(i));
        Assert.assertEquals(invertedSortIndex.get(i), actualInvertedSortIndex.get(i));
    }
}

Example 3 with CarbonDictionaryWriterImpl

use of org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl in project carbondata by apache.

the class CarbonDictionarySortIndexWriterImplTest method setUp.

@Before
public void setUp() throws Exception {
    storePath = "target/carbonStore";
    carbonTableIdentifier = new CarbonTableIdentifier("testSchema", "carbon", UUID.randomUUID().toString());
    columnIdentifier = new ColumnIdentifier("Name", null, null);
    dictionaryWriter = new CarbonDictionaryWriterImpl(storePath, carbonTableIdentifier, columnIdentifier);
    dictionarySortIndexWriter = new CarbonDictionarySortIndexWriterImpl(carbonTableIdentifier, columnIdentifier, storePath);
    carbonDictionarySortIndexReader = new CarbonDictionarySortIndexReaderImpl(carbonTableIdentifier, columnIdentifier, storePath);
}

Example 4 with CarbonDictionaryWriterImpl

use of org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl in project carbondata by apache.

the class StoreCreator method writeDictionary.

private static void writeDictionary(String factFilePath, CarbonTable table) throws Exception {
    BufferedReader reader = new BufferedReader(new FileReader(factFilePath));
    String header = reader.readLine();
    String[] split = header.split(",");
    List<CarbonColumn> allCols = new ArrayList<CarbonColumn>();
    List<CarbonDimension> dims = table.getDimensionByTableName(table.getFactTableName());
    allCols.addAll(dims);
    List<CarbonMeasure> msrs = table.getMeasureByTableName(table.getFactTableName());
    allCols.addAll(msrs);
    Set<String>[] set = new HashSet[dims.size()];
    for (int i = 0; i < set.length; i++) {
        set[i] = new HashSet<String>();
    }
    String line = reader.readLine();
    while (line != null) {
        String[] data = line.split(",");
        for (int i = 0; i < set.length; i++) {
            set[i].add(data[i]);
        }
        line = reader.readLine();
    }
    Cache dictCache = CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY, absoluteTableIdentifier.getStorePath());
    for (int i = 0; i < set.length; i++) {
        ColumnIdentifier columnIdentifier = new ColumnIdentifier(dims.get(i).getColumnId(), null, null);
        CarbonDictionaryWriter writer = new CarbonDictionaryWriterImpl(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier(), columnIdentifier);
        for (String value : set[i]) {
            writer.write(value);
        }
        writer.close();
        writer.commit();
        Dictionary dict = (Dictionary) dictCache.get(new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier.getCarbonTableIdentifier(), columnIdentifier, dims.get(i).getDataType()));
        CarbonDictionarySortInfoPreparator preparator = new CarbonDictionarySortInfoPreparator();
        List<String> newDistinctValues = new ArrayList<String>();
        CarbonDictionarySortInfo dictionarySortInfo = preparator.getDictionarySortInfo(newDistinctValues, dict, dims.get(i).getDataType());
        CarbonDictionarySortIndexWriter carbonDictionaryWriter = new CarbonDictionarySortIndexWriterImpl(absoluteTableIdentifier.getCarbonTableIdentifier(), columnIdentifier, absoluteTableIdentifier.getStorePath());
        try {
            carbonDictionaryWriter.writeSortIndex(dictionarySortInfo.getSortIndex());
            carbonDictionaryWriter.writeInvertedSortIndex(dictionarySortInfo.getSortIndexInverted());
        } finally {
            carbonDictionaryWriter.close();
        }
    }
    reader.close();
}

Example 5 with CarbonDictionaryWriterImpl

use of org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl in project carbondata by apache.

the class StoreCreator method writeDictionary.

private static void writeDictionary(String factFilePath, CarbonTable table) throws Exception {
    BufferedReader reader = new BufferedReader(new FileReader(factFilePath));
    String header = reader.readLine();
    String[] split = header.split(",");
    List<CarbonColumn> allCols = new ArrayList<CarbonColumn>();
    List<CarbonDimension> dims = table.getDimensionByTableName(table.getFactTableName());
    allCols.addAll(dims);
    List<CarbonMeasure> msrs = table.getMeasureByTableName(table.getFactTableName());
    allCols.addAll(msrs);
    Set<String>[] set = new HashSet[dims.size()];
    for (int i = 0; i < set.length; i++) {
        set[i] = new HashSet<String>();
    }
    String line = reader.readLine();
    while (line != null) {
        String[] data = line.split(",");
        for (int i = 0; i < set.length; i++) {
            set[i].add(data[i]);
        }
        line = reader.readLine();
    }
    Cache dictCache = CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY, absoluteTableIdentifier.getStorePath());
    for (int i = 0; i < set.length; i++) {
        ColumnIdentifier columnIdentifier = new ColumnIdentifier(dims.get(i).getColumnId(), null, null);
        CarbonDictionaryWriter writer = new CarbonDictionaryWriterImpl(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier(), columnIdentifier);
        for (String value : set[i]) {
            writer.write(value);
        }
        writer.close();
        writer.commit();
        Dictionary dict = (Dictionary) dictCache.get(new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier.getCarbonTableIdentifier(), columnIdentifier, dims.get(i).getDataType()));
        CarbonDictionarySortInfoPreparator preparator = new CarbonDictionarySortInfoPreparator();
        List<String> newDistinctValues = new ArrayList<String>();
        CarbonDictionarySortInfo dictionarySortInfo = preparator.getDictionarySortInfo(newDistinctValues, dict, dims.get(i).getDataType());
        CarbonDictionarySortIndexWriter carbonDictionaryWriter = new CarbonDictionarySortIndexWriterImpl(absoluteTableIdentifier.getCarbonTableIdentifier(), columnIdentifier, absoluteTableIdentifier.getStorePath());
        try {
            carbonDictionaryWriter.writeSortIndex(dictionarySortInfo.getSortIndex());
            carbonDictionaryWriter.writeInvertedSortIndex(dictionarySortInfo.getSortIndexInverted());
        } finally {
            carbonDictionaryWriter.close();
        }
    }
    reader.close();
}