Search in sources :

Example 1 with Bucket

use of com.bakdata.conquery.models.events.Bucket in project conquery by bakdata.

the class SecondaryIdQueryPlan method executeQueriesWithoutSecondaryId.

private void executeQueriesWithoutSecondaryId(QueryExecutionContext ctx, Entity entity, Table currentTable) {
    nextTable(ctx, currentTable);
    final List<Bucket> tableBuckets = ctx.getBucketManager().getEntityBucketsForTable(entity, currentTable);
    for (Bucket bucket : tableBuckets) {
        int entityId = entity.getId();
        nextBlock(bucket);
        if (!bucket.containsEntity(entityId) || !isOfInterest(bucket)) {
            continue;
        }
        int start = bucket.getEntityStart(entityId);
        int end = bucket.getEntityEnd(entityId);
        for (int event = start; event < end; event++) {
            for (ConceptQueryPlan child : childPerKey.values()) {
                child.nextEvent(bucket, event);
            }
        }
    }
}
Also used : Bucket(com.bakdata.conquery.models.events.Bucket)

Example 2 with Bucket

use of com.bakdata.conquery.models.events.Bucket in project conquery by bakdata.

the class TableExportQueryPlan method execute.

@Override
public Optional<MultilineEntityResult> execute(QueryExecutionContext ctx, Entity entity) {
    Optional<? extends EntityResult> result = subPlan.execute(ctx, entity);
    if (result.isEmpty() || tables.isEmpty()) {
        return Optional.empty();
    }
    List<Object[]> results = new ArrayList<>();
    final int totalColumns = positions.values().stream().mapToInt(i -> i).max().getAsInt() + 1;
    for (TableExportDescription exportDescription : tables) {
        for (Bucket bucket : ctx.getEntityBucketsForTable(entity, exportDescription.getTable())) {
            int entityId = entity.getId();
            if (!bucket.containsEntity(entityId)) {
                continue;
            }
            int start = bucket.getEntityStart(entityId);
            int end = bucket.getEntityEnd(entityId);
            for (int event = start; event < end; event++) {
                // Export Full-table if it has no validity date.
                if (exportDescription.getValidityDateColumn() != null && !bucket.eventIsContainedIn(event, exportDescription.getValidityDateColumn(), CDateSet.create(dateRange))) {
                    continue;
                }
                Object[] entry = new Object[totalColumns];
                for (Column column : exportDescription.getTable().getColumns()) {
                    if (!bucket.has(event, column)) {
                        continue;
                    }
                    if (column.equals(exportDescription.getValidityDateColumn())) {
                        entry[0] = List.of(bucket.getAsDateRange(event, column));
                    } else {
                        entry[positions.get(column)] = bucket.createScriptValue(event, column);
                    }
                }
                results.add(entry);
            }
        }
    }
    return Optional.of(new MultilineEntityResult(entity.getId(), results));
}
Also used : Bucket(com.bakdata.conquery.models.events.Bucket) Getter(lombok.Getter) Table(com.bakdata.conquery.models.datasets.Table) EntityResult(com.bakdata.conquery.models.query.results.EntityResult) RequiredArgsConstructor(lombok.RequiredArgsConstructor) CDateRange(com.bakdata.conquery.models.common.daterange.CDateRange) MultilineEntityResult(com.bakdata.conquery.models.query.results.MultilineEntityResult) ArrayList(java.util.ArrayList) Entity(com.bakdata.conquery.models.query.entity.Entity) List(java.util.List) QueryExecutionContext(com.bakdata.conquery.models.query.QueryExecutionContext) Map(java.util.Map) Aggregator(com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator) Optional(java.util.Optional) Nullable(javax.annotation.Nullable) CDateSet(com.bakdata.conquery.models.common.CDateSet) Column(com.bakdata.conquery.models.datasets.Column) Bucket(com.bakdata.conquery.models.events.Bucket) Column(com.bakdata.conquery.models.datasets.Column) ArrayList(java.util.ArrayList) MultilineEntityResult(com.bakdata.conquery.models.query.results.MultilineEntityResult)

Example 3 with Bucket

use of com.bakdata.conquery.models.events.Bucket in project conquery by bakdata.

the class ImportJob method sendBuckets.

/**
 * select, then send buckets.
 */
private Map<WorkerId, Set<BucketId>> sendBuckets(Map<Integer, Integer> starts, Map<Integer, Integer> lengths, DictionaryMapping primaryMapping, Import imp, Map<Integer, List<Integer>> buckets2LocalEntities, ColumnStore[] storesSorted) throws JsonProcessingException {
    Map<WorkerId, Set<BucketId>> newWorkerAssignments = new HashMap<>();
    final ProgressReporter subJob = getProgressReporter().subJob(buckets2LocalEntities.size());
    for (Map.Entry<Integer, List<Integer>> bucket2entities : buckets2LocalEntities.entrySet()) {
        WorkerInformation responsibleWorker = Objects.requireNonNull(namespace.getResponsibleWorkerForBucket(bucket2entities.getKey()), () -> "No responsible worker for Bucket#" + bucket2entities.getKey());
        awaitFreeJobQueue(responsibleWorker);
        final Bucket bucket = selectBucket(starts, lengths, storesSorted, primaryMapping, imp, bucket2entities.getKey(), bucket2entities.getValue());
        newWorkerAssignments.computeIfAbsent(responsibleWorker.getId(), (ignored) -> new HashSet<>()).add(bucket.getId());
        log.trace("Sending Bucket[{}] to {}", bucket.getId(), responsibleWorker.getId());
        responsibleWorker.send(ImportBucket.forBucket(bucket));
        subJob.report(1);
    }
    subJob.done();
    return newWorkerAssignments;
}
Also used : Dictionary(com.bakdata.conquery.models.dictionary.Dictionary) ColumnStore(com.bakdata.conquery.models.events.stores.root.ColumnStore) java.util(java.util) ConqueryConfig(com.bakdata.conquery.models.config.ConqueryConfig) Getter(lombok.Getter) PreprocessedHeader(com.bakdata.conquery.models.preproc.PreprocessedHeader) RequiredArgsConstructor(lombok.RequiredArgsConstructor) PreprocessedData(com.bakdata.conquery.models.preproc.PreprocessedData) com.bakdata.conquery.models.identifiable.ids.specific(com.bakdata.conquery.models.identifiable.ids.specific) NamespaceStorage(com.bakdata.conquery.io.storage.NamespaceStorage) IdMutex(com.bakdata.conquery.models.identifiable.IdMutex) com.bakdata.conquery.models.datasets(com.bakdata.conquery.models.datasets) IntegerStore(com.bakdata.conquery.models.events.stores.root.IntegerStore) WorkerInformation(com.bakdata.conquery.models.worker.WorkerInformation) BadRequestException(javax.ws.rs.BadRequestException) JSONException(com.bakdata.conquery.models.exceptions.JSONException) DictionaryMapping(com.bakdata.conquery.models.dictionary.DictionaryMapping) IntegerParser(com.bakdata.conquery.models.preproc.parser.specific.IntegerParser) PreprocessedReader(com.bakdata.conquery.models.preproc.PreprocessedReader) ResourceUtil(com.bakdata.conquery.util.ResourceUtil) Bucket(com.bakdata.conquery.models.events.Bucket) MajorTypeId(com.bakdata.conquery.models.events.MajorTypeId) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) IOException(java.io.IOException) StringStore(com.bakdata.conquery.models.events.stores.root.StringStore) com.bakdata.conquery.models.messages.namespaces.specific(com.bakdata.conquery.models.messages.namespaces.specific) Collectors(java.util.stream.Collectors) Entity(com.bakdata.conquery.models.query.entity.Entity) ProgressReporter(com.bakdata.conquery.util.progressreporter.ProgressReporter) Slf4j(lombok.extern.slf4j.Slf4j) IntList(it.unimi.dsi.fastutil.ints.IntList) Response(javax.ws.rs.core.Response) IntOpenHashSet(it.unimi.dsi.fastutil.ints.IntOpenHashSet) WebApplicationException(javax.ws.rs.WebApplicationException) IntSet(it.unimi.dsi.fastutil.ints.IntSet) PreprocessedDictionaries(com.bakdata.conquery.models.preproc.PreprocessedDictionaries) IntArrayList(it.unimi.dsi.fastutil.ints.IntArrayList) Namespace(com.bakdata.conquery.models.worker.Namespace) InputStream(java.io.InputStream) WorkerInformation(com.bakdata.conquery.models.worker.WorkerInformation) IntOpenHashSet(it.unimi.dsi.fastutil.ints.IntOpenHashSet) IntSet(it.unimi.dsi.fastutil.ints.IntSet) ProgressReporter(com.bakdata.conquery.util.progressreporter.ProgressReporter) Bucket(com.bakdata.conquery.models.events.Bucket) IntList(it.unimi.dsi.fastutil.ints.IntList) IntArrayList(it.unimi.dsi.fastutil.ints.IntArrayList) IntOpenHashSet(it.unimi.dsi.fastutil.ints.IntOpenHashSet)

Example 4 with Bucket

use of com.bakdata.conquery.models.events.Bucket in project conquery by bakdata.

the class ConceptQueryPlan method execute.

@Override
public Optional<SinglelineEntityResult> execute(QueryExecutionContext ctx, Entity entity) {
    // Only override if none has been set from a higher level
    ctx = QueryUtils.determineDateAggregatorForContext(ctx, this::getValidityDateAggregator);
    if (!isOfInterest(entity)) {
        return Optional.empty();
    }
    // Always do one go-round with ALL_IDS_TABLE.
    nextTable(ctx, ctx.getStorage().getDataset().getAllIdsTable());
    nextBlock(EmptyBucket.getInstance());
    nextEvent(EmptyBucket.getInstance(), 0);
    for (Table currentTable : requiredTables.get()) {
        if (Dataset.isAllIdsTable(currentTable)) {
            continue;
        }
        nextTable(ctx, currentTable);
        final List<Bucket> tableBuckets = ctx.getBucketManager().getEntityBucketsForTable(entity, currentTable);
        log.trace("Table[{}] has {} buckets for Entity[{}]", currentTable, tableBuckets, entity);
        for (Bucket bucket : tableBuckets) {
            if (!isOfInterest(bucket)) {
                continue;
            }
            nextBlock(bucket);
            int start = bucket.getEntityStart(entity.getId());
            int end = bucket.getEntityEnd(entity.getId());
            for (int event = start; event < end; event++) {
                nextEvent(bucket, event);
            }
        }
    }
    if (isContained()) {
        return Optional.of(createResult());
    }
    return Optional.empty();
}
Also used : Table(com.bakdata.conquery.models.datasets.Table) Bucket(com.bakdata.conquery.models.events.Bucket) EmptyBucket(com.bakdata.conquery.models.events.EmptyBucket)

Example 5 with Bucket

use of com.bakdata.conquery.models.events.Bucket in project conquery by bakdata.

the class SecondaryIdQueryPlan method executeQueriesWithSecondaryId.

private void executeQueriesWithSecondaryId(QueryExecutionContext ctx, Entity entity, Column secondaryIdColumnId) {
    QueryExecutionContext ctxWithPhase = ctx.withActiveSecondaryId(getSecondaryId());
    Table currentTable = secondaryIdColumnId.getTable();
    nextTable(ctxWithPhase, currentTable);
    final List<Bucket> tableBuckets = ctx.getBucketManager().getEntityBucketsForTable(entity, currentTable);
    for (Bucket bucket : tableBuckets) {
        int entityId = entity.getId();
        nextBlock(bucket);
        if (!bucket.containsEntity(entityId)) {
            continue;
        }
        if (!isOfInterest(bucket)) {
            continue;
        }
        int start = bucket.getEntityStart(entityId);
        int end = bucket.getEntityEnd(entityId);
        for (int event = start; event < end; event++) {
            // we ignore events with no value in the secondaryIdColumn
            if (!bucket.has(event, secondaryIdColumnId)) {
                continue;
            }
            String key = ((String) bucket.createScriptValue(event, secondaryIdColumnId));
            final ConceptQueryPlan plan = childPerKey.computeIfAbsent(key, k -> createChild(ctxWithPhase, bucket));
            plan.nextEvent(bucket, event);
        }
    }
}
Also used : Table(com.bakdata.conquery.models.datasets.Table) QueryExecutionContext(com.bakdata.conquery.models.query.QueryExecutionContext) Bucket(com.bakdata.conquery.models.events.Bucket)

Aggregations

Bucket (com.bakdata.conquery.models.events.Bucket)7 Table (com.bakdata.conquery.models.datasets.Table)4 ColumnStore (com.bakdata.conquery.models.events.stores.root.ColumnStore)3 Entity (com.bakdata.conquery.models.query.entity.Entity)3 NamespaceStorage (com.bakdata.conquery.io.storage.NamespaceStorage)2 ConqueryConfig (com.bakdata.conquery.models.config.ConqueryConfig)2 com.bakdata.conquery.models.datasets (com.bakdata.conquery.models.datasets)2 Column (com.bakdata.conquery.models.datasets.Column)2 Dictionary (com.bakdata.conquery.models.dictionary.Dictionary)2 DictionaryMapping (com.bakdata.conquery.models.dictionary.DictionaryMapping)2 MajorTypeId (com.bakdata.conquery.models.events.MajorTypeId)2 IntegerStore (com.bakdata.conquery.models.events.stores.root.IntegerStore)2 StringStore (com.bakdata.conquery.models.events.stores.root.StringStore)2 JSONException (com.bakdata.conquery.models.exceptions.JSONException)2 IdMutex (com.bakdata.conquery.models.identifiable.IdMutex)2 com.bakdata.conquery.models.identifiable.ids.specific (com.bakdata.conquery.models.identifiable.ids.specific)2 com.bakdata.conquery.models.messages.namespaces.specific (com.bakdata.conquery.models.messages.namespaces.specific)2 PreprocessedData (com.bakdata.conquery.models.preproc.PreprocessedData)2 PreprocessedDictionaries (com.bakdata.conquery.models.preproc.PreprocessedDictionaries)2 PreprocessedHeader (com.bakdata.conquery.models.preproc.PreprocessedHeader)2