Examples with EncodedDictionary com.bakdata.conquery.models.dictionary.EncodedDictionary used on opensource projects

Example 1 with EncodedDictionary

use of com.bakdata.conquery.models.dictionary.EncodedDictionary in project conquery by bakdata.

the class BigStoreTest method testFull.

@Test
public void testFull() throws JSONException, IOException {
    BigStore<DictionaryId, Dictionary> store = new BigStore<>(new XodusStoreFactory(), Validators.newValidator(), env, StoreMappings.DICTIONARIES.storeInfo(), (e) -> {
    }, (e) -> {
    }, MAPPER);
    store.setChunkByteSize(Ints.checkedCast(DataSize.megabytes(1).toBytes()));
    Dictionary nDict = new MapDictionary(Dataset.PLACEHOLDER, "dict");
    for (int v = 0; v < 1000000; v++) {
        nDict.add(Integer.toHexString(v).getBytes());
    }
    // check if manual serialization deserialization works
    byte[] bytes = Jackson.BINARY_MAPPER.writeValueAsBytes(nDict);
    Dictionary simpleCopy = MAPPER.readValue(bytes, Dictionary.class);
    for (int v = 0; v < 1000000; v++) {
        assertThat(simpleCopy.getId(Integer.toHexString(v).getBytes())).isEqualTo(v);
    }
    // check if store works
    store.add(nDict.getId(), nDict);
    // check if the bytes in the store are the same as bytes
    assertThat(new SequenceInputStream(Iterators.asEnumeration(store.getMetaStore().get(nDict.getId()).loadData(store.getDataStore()).map(ByteArrayInputStream::new).iterator()))).hasSameContentAs(new ByteArrayInputStream(bytes));
    EncodedDictionary copy = new EncodedDictionary(store.get(nDict.getId()), StringTypeEncoded.Encoding.UTF8);
    for (int v = 0; v < 1000000; v++) {
        assertThat(copy.getId(Integer.toHexString(v))).isEqualTo(v);
    }
}

Example 2 with EncodedDictionary

use of com.bakdata.conquery.models.dictionary.EncodedDictionary in project conquery by bakdata.

the class SuccinctTrieTest method valid.

@ParameterizedTest(name = "seed: {0}")
@MethodSource("getSeeds")
public void valid(long seed) {
    final SuccinctTrie dict = new SuccinctTrie(Dataset.PLACEHOLDER, "name");
    EncodedDictionary direct = new EncodedDictionary(dict, StringTypeEncoded.Encoding.UTF8);
    final BiMap<String, Integer> reference = HashBiMap.create();
    AtomicInteger count = new AtomicInteger(0);
    Random random = new Random(seed);
    IntStream.range(0, 8192).boxed().sorted(TernaryTreeTestUtil.shuffle(random)).forEach(rep -> {
        final String prefix = Integer.toString(rep, 26);
        reference.put(prefix, count.get());
        dict.add(prefix.getBytes());
        count.incrementAndGet();
    });
    log.info("structure build");
    dict.compress();
    log.info("trie compressed");
    // assert key value lookup
    assertThat(reference.entrySet().stream()).allSatisfy(entry -> {
        assertThat(direct.getId(entry.getKey())).isEqualTo(entry.getValue());
    });
    log.info("forward lookup done");
    // assert reverse lookup
    assertThat(reference.inverse().entrySet().stream()).allSatisfy(entry -> {
        assertThat(dict.getElement(entry.getKey())).isEqualTo(entry.getValue().getBytes());
    });
    log.info("reverse lookup done");
}