use of com.bakdata.conquery.models.events.stores.specific.string.StringTypeEncoded in project conquery by bakdata.
the class MapTypeGuesser method createGuess.
@Override
public Guess createGuess() {
IntegerStore indexType = p.decideIndexType();
StringTypeDictionary type = new StringTypeDictionary(indexType, null);
long mapSize = MapDictionary.estimateMemoryConsumption(p.getStrings().size(), p.getDecoded().stream().mapToLong(s -> s.length).sum());
StringTypeEncoded result = new StringTypeEncoded(type, p.getEncoding());
return new Guess(result, indexType.estimateMemoryConsumptionBytes(), mapSize) {
@Override
public StringStore getType() {
MapDictionary map = new MapDictionary(Dataset.PLACEHOLDER, UUID.randomUUID().toString());
for (byte[] v : p.getDecoded()) {
map.add(v);
}
type.setDictionary(map);
return super.getType();
}
};
}
use of com.bakdata.conquery.models.events.stores.specific.string.StringTypeEncoded in project conquery by bakdata.
the class TrieTypeGuesser method createGuess.
@Override
public Guess createGuess() {
IntegerStore indexType = p.decideIndexType();
SuccinctTrie trie = new SuccinctTrie(Dataset.PLACEHOLDER, UUID.randomUUID().toString());
StringTypeDictionary type = new StringTypeDictionary(indexType, trie);
for (byte[] v : p.getDecoded()) {
trie.add(v);
}
StringTypeEncoded result = new StringTypeEncoded(type, p.getEncoding());
return new Guess(result, indexType.estimateMemoryConsumptionBytes(), trie.estimateMemoryConsumption()) {
@Override
public StringStore getType() {
trie.compress();
return super.getType();
}
};
}
use of com.bakdata.conquery.models.events.stores.specific.string.StringTypeEncoded in project conquery by bakdata.
the class StringEncodingTest method testHexStreamStringType.
@Test
public void testHexStreamStringType() {
StringParser parser = new StringParser(new ConqueryConfig());
Stream.generate(() -> UUID.randomUUID().toString().replace("-", "")).map(String::toUpperCase).mapToInt(v -> {
try {
return parser.parse(v);
} catch (ParsingException e) {
// We know that StringTypeVarInt is able to parse our strings.
return 0;
}
}).limit(100).forEach(parser::addLine);
StringTypeEncoded subType = (StringTypeEncoded) parser.findBestType();
assertThat(subType).isInstanceOf(StringTypeEncoded.class);
assertThat(subType.getEncoding()).isEqualByComparingTo(StringTypeEncoded.Encoding.Base16UpperCase);
}
Aggregations