Examples with Bucket com.bakdata.conquery.models.events.Bucket used on opensource projects

Example 6 with Bucket

use of com.bakdata.conquery.models.events.Bucket in project conquery by bakdata.

the class SerializationTests method bucketCompoundDateRange.

@Test
public void bucketCompoundDateRange() throws JSONException, IOException {
    Dataset dataset = new Dataset();
    dataset.setName("datasetName");
    Table table = new Table();
    Column startCol = new Column();
    startCol.setName("startCol");
    startCol.setType(MajorTypeId.DATE);
    startCol.setTable(table);
    Column endCol = new Column();
    endCol.setLabel("endLabel");
    endCol.setName("endCol");
    endCol.setType(MajorTypeId.DATE);
    endCol.setTable(table);
    Column compoundCol = new Column();
    compoundCol.setName("compoundCol");
    compoundCol.setType(MajorTypeId.DATE_RANGE);
    compoundCol.setTable(table);
    table.setColumns(new Column[] { startCol, endCol, compoundCol });
    table.setDataset(dataset);
    table.setName("tableName");
    Import imp = new Import(table);
    imp.setName("importTest");
    DateRangeTypeCompound compoundStore = new DateRangeTypeCompound(startCol.getName(), endCol.getName(), new BitSetStore(BitSet.valueOf(new byte[] { 0b1000 }), new BitSet(), 4));
    // 0b1000 is a binary representation of 8 so that the 4th is set to make sure that BitSet length is 4.
    ColumnStore startStore = new IntegerDateStore(new ShortArrayStore(new short[] { 1, 2, 3, 4 }, Short.MIN_VALUE));
    ColumnStore endStore = new IntegerDateStore(new ShortArrayStore(new short[] { 5, 6, 7, 8 }, Short.MIN_VALUE));
    Bucket bucket = new Bucket(0, 1, 4, new ColumnStore[] { startStore, endStore, compoundStore }, Collections.emptySet(), new int[0], new int[0], imp);
    compoundStore.setParent(bucket);
    CentralRegistry registry = new CentralRegistry();
    registry.register(dataset);
    registry.register(startCol);
    registry.register(endCol);
    registry.register(compoundCol);
    registry.register(table);
    registry.register(imp);
    registry.register(bucket);
    final Validator validator = Validators.newValidator();
    SerializationTestUtil.forType(Bucket.class).registry(registry).injectables(new Injectable() {

        @Override
        public MutableInjectableValues inject(MutableInjectableValues values) {
            return values.add(Validator.class, validator);
        }
    }).test(bucket);
}

Example 7 with Bucket

use of com.bakdata.conquery.models.events.Bucket in project conquery by bakdata.

the class ImportJob method selectBucket.

/**
 * - remap Entity-Ids to global
 * - calculate per-Entity regions of Bucklet (start/end)
 * - split stores
 */
private Bucket selectBucket(Map<Integer, Integer> localStarts, Map<Integer, Integer> localLengths, ColumnStore[] stores, DictionaryMapping primaryMapping, Import imp, int bucketId, List<Integer> localEntities) {
    final int root = bucketSize * bucketId;
    IntList selectionStart = new IntArrayList();
    IntList selectionLength = new IntArrayList();
    IntSet entities = new IntOpenHashSet();
    // First entity of Bucket starts at 0, the following are appended.
    int[] entityStarts = new int[bucketSize];
    int[] entityEnds = new int[bucketSize];
    Arrays.fill(entityEnds, -1);
    Arrays.fill(entityStarts, -1);
    int currentStart = 0;
    for (int position = 0; position < bucketSize; position++) {
        int globalId = root + position;
        int localId = primaryMapping.target2Source(globalId);
        if (localId == -1) {
            continue;
        }
        if (!localStarts.containsKey(localId)) {
            continue;
        }
        entities.add(globalId);
        final int length = localLengths.get(localId);
        selectionStart.add(localStarts.get(localId));
        selectionLength.add(length);
        entityStarts[position] = currentStart;
        entityEnds[position] = currentStart + length;
        currentStart += length;
    }
    // copy only the parts of the bucket we need
    final ColumnStore[] bucketStores = Arrays.stream(stores).map(store -> store.select(selectionStart.toIntArray(), selectionLength.toIntArray())).toArray(ColumnStore[]::new);
    return new Bucket(bucketId, root, selectionLength.intStream().sum(), bucketStores, entities, entityStarts, entityEnds, imp);
}