Examples with StringDoublePair org.vitrivr.cineast.core.data.StringDoublePair used on opensource projects

Example 1 with StringDoublePair

use of org.vitrivr.cineast.core.data.StringDoublePair in project cineast by vitrivr.

the class CineastQueryService method getSimilar.

// TODO This has enormous code duplication with the TemporalQueryMessageHandler
@Override
public void getSimilar(CineastGrpc.TemporalQuery query, StreamObserver<CineastGrpc.QueryResult> responseObserver) {
    StopWatch watch = StopWatch.createStarted();
    MediaSegmentReader mediaSegmentReader = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    MediaObjectReader mediaObjectReader = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    MediaSegmentMetadataReader segmentMetadataReader = new MediaSegmentMetadataReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    MediaObjectMetadataReader objectMetadataReader = new MediaObjectMetadataReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    Set<String> sentSegmentIds = new HashSet<>(), sentObjectIds = new HashSet<>();
    CineastGrpc.QueryConfig config = query.getQueryList().get(0).getConfig();
    ReadableQueryConfig rqconf = QueryContainerUtil.queryConfig(config);
    QueryConfig qconf = new QueryConfig(rqconf);
    /* Prepare QueryConfig (so as to obtain a QueryId). */
    final String uuid = qconf.getQueryId().toString();
    final int max = qconf.getMaxResults().orElse(Config.sharedConfig().getRetriever().getMaxResults());
    qconf.setMaxResults(max);
    final int resultsPerModule = qconf.getRawResultsPerModule() == -1 ? Config.sharedConfig().getRetriever().getMaxResultsPerModule() : qconf.getResultsPerModule();
    qconf.setResultsPerModule(resultsPerModule);
    List<Thread> metadataRetrievalThreads = new ArrayList<>();
    /* We iterate over all components independently, because they have a temporal context.*/
    for (int containerIdx = 0; containerIdx < query.getQueryCount(); containerIdx++) {
        List<QueryStage> stages = QueryContainerUtil.query(query.getQueryList().get(containerIdx));
        /* We make a new stagedQueryConfig per stage because the relevant segments will differ for each stage. This also resets the filter (relevant ids in the config)*/
        QueryConfig stageQConf = QueryConfig.clone(qconf);
        /* For the first stage, there will be no relevant segments when querying. This is ok because the retrieval engine handles this appropriately */
        HashSet<String> relevantSegments = new HashSet<>();
        /* Store for each queryterm per category all results to be sent at a later time */
        List<Map<String, List<StringDoublePair>>> cache = new ArrayList<>();
        /* For the terms of a stage, ordering matters. The assumption is that each term is used as a filter for its successor */
        for (int stageIndex = 0; stageIndex < stages.size(); stageIndex++) {
            /* Initalize stage with this hashmap */
            cache.add(stageIndex, new HashMap<>());
            QueryStage stage = stages.get(stageIndex);
            List<Thread> qtThreads = new ArrayList<>();
            /* We now iterate over all QueryTerms for this stage, simply adding their results to the list of relevant segments for the next querystage.
         * The list is only updated once we've iterated over all terms
         */
            for (int i = 0; i < stage.getQueryTerms().size(); i++) {
                QueryTerm qt = stage.getQueryTerms().get(i);
                final int finalContainerIdx = containerIdx;
                final int finalStageIndex = stageIndex;
                Thread qtRetrievalThread = new Thread(() -> {
                    /* Prepare QueryTerm and perform sanity-checks */
                    if (qt == null) {
                        /* In rare instances, it is possible to have null as query stage. If this happens to you, please report this to the developers so we can try to fix it. */
                        LOGGER.warn("QueryTerm was null for stage {}", stage);
                        return;
                    }
                    AbstractQueryTermContainer qc = qt.getContainer();
                    if (qc == null) {
                        LOGGER.warn("Likely an empty query, as it could not be converted to a query container. Ignoring it");
                        return;
                    }
                    List<Thread> categoryThreads = new ArrayList<>();
                    /* For each category of a specific queryterm, we actually go and retrieve. Be aware that we do not change the relevant ids after this call */
                    for (String category : qt.getCategories()) {
                        /* Merge partial results with score-map */
                        List<SegmentScoreElement> scores = continuousRetrievalLogic.retrieve(qc, category, stageQConf);
                        /* Transform raw results into list of StringDoublePairs (segmentId -> score) */
                        final List<StringDoublePair> results = scores.stream().map(elem -> new StringDoublePair(elem.getSegmentId(), elem.getScore())).filter(p -> p.value > 0d).sorted(StringDoublePair.COMPARATOR).limit(max).collect(Collectors.toList());
                        if (results.isEmpty()) {
                            LOGGER.warn("No results found for category {} and qt {} in stage with id {}. Full compoment: {}", category, qt, finalContainerIdx, stage);
                        }
                        if (cache.get(finalStageIndex).containsKey(category)) {
                            LOGGER.error("Category {} was used twice in stage {}. This erases the results of the previous category... ", category, finalStageIndex);
                        }
                        cache.get(finalStageIndex).put(category, results);
                        results.forEach(res -> relevantSegments.add(res.key));
                        LOGGER.trace("Category {} at stage {} executed @ {} ms", category, finalStageIndex, watch.getTime(TimeUnit.MILLISECONDS));
                        /* If this is the last stage, we can send relevant results per category back to the UI.
               * Otherwise, we cannot since we might send results to the UI which would be filtered at a later stage
               */
                        if (finalStageIndex == stages.size() - 1) {
                            /* Finalize and submit per-container results */
                            responseObserver.onNext(QueryContainerUtil.queryResult(QueryContainerUtil.similarityQueryResult(qt.getQueryConfig().getQueryId().toString(), category, results)));
                            List<String> segmentIds = results.stream().map(x -> x.key).filter(x -> !sentSegmentIds.contains(x)).collect(Collectors.toList());
                            if (segmentIds.isEmpty()) {
                                continue;
                            }
                            Map<String, MediaSegmentDescriptor> segments = mediaSegmentReader.lookUpSegments(segmentIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaSegmentQueryResult.newBuilder().addAllSegments(segments.values().stream().map(MediaSegmentUtil::fromMediaSegmentDescriptor).collect(Collectors.toList())).build()));
                            List<MediaSegmentMetadataDescriptor> segmentMetaData = segmentMetadataReader.lookupMultimediaMetadata(segmentIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaSegmentMetaDataQueryResult.newBuilder().addAllSegmentMetaData(segmentMetaData.stream().map(QueryContainerUtil::mediaSegmentMetaData).collect(Collectors.toList())).build()));
                            sentSegmentIds.addAll(segmentIds);
                            List<String> objectIds = segments.values().stream().map(MediaSegmentDescriptor::getObjectId).filter(x -> !sentObjectIds.contains(x)).collect(Collectors.toList());
                            if (objectIds.isEmpty()) {
                                continue;
                            }
                            Map<String, MediaObjectDescriptor> objects = mediaObjectReader.lookUpObjects(objectIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaObjectQueryResult.newBuilder().addAllObjects(objects.values().stream().map(MediaObjectUtil::fromMediaObjectDescriptor).collect(Collectors.toList())).build()));
                            List<MediaObjectMetadataDescriptor> objectMetaData = objectMetadataReader.lookupMultimediaMetadata(objectIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaObjectMetaDataQueryResult.newBuilder().addAllObjectMetaData(objectMetaData.stream().map(QueryContainerUtil::mediaObjectMetaData).collect(Collectors.toList())).build()));
                            sentObjectIds.addAll(objectIds);
                        }
                    }
                /* We're done for this querycontainer */
                });
                // TODO Better name
                qtRetrievalThread.setName("qt-stage" + stageIndex + "-" + qt.getCategories());
                qtThreads.add(qtRetrievalThread);
                qtRetrievalThread.start();
            }
            for (Thread thread : qtThreads) {
                try {
                    thread.join();
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
            /* After we are done with a stage, we add all relevant segments to the config for the next stage. */
            if (relevantSegments.size() == 0) {
                LOGGER.warn("No relevant segments anymore, aborting staged querying");
                /* Clear relevant segments (there are none) */
                stageQConf.setRelevantSegmentIds(relevantSegments);
                break;
            }
            stageQConf.setRelevantSegmentIds(relevantSegments);
            relevantSegments.clear();
        }
        /* At this point, we have iterated over all stages. Now, we need to go back for all stages and send the results for the relevant ids. */
        for (int stageIndex = 0; stageIndex < stages.size() - 1; stageIndex++) {
            cache.get(stageIndex).forEach((category, results) -> {
                results.removeIf(pair -> !stageQConf.getRelevantSegmentIds().contains(pair.key));
                responseObserver.onNext(QueryContainerUtil.queryResult(QueryContainerUtil.similarityQueryResult(// TODO This assumes that all queries in a temporalquery have the same uuid
                uuid, category, results)));
            });
        }
    /* There should be no carry-over from this block since temporal queries are executed independently */
    }
    /* At this point, all StagedQueries have been executed for this TemporalQuery.
     * Since results have always been sent for the final stage or, when appropriate, in intermediate steps, there's nothing left to do.
     */
    responseObserver.onCompleted();
    mediaSegmentReader.close();
    mediaObjectReader.close();
    segmentMetadataReader.close();
    watch.stop();
    LOGGER.debug("Query executed in {} ms", watch.getTime(TimeUnit.MILLISECONDS));
}

Example 2 with StringDoublePair

use of org.vitrivr.cineast.core.data.StringDoublePair in project cineast by vitrivr.

the class TemporalTestCases method buildTestCase4.

public void buildTestCase4() {
    Map<String, MediaSegmentDescriptor> segmentMap = new HashMap<>();
    List<List<StringDoublePair>> containerResults = new ArrayList<>();
    segmentMap.put(descriptor1_1.getSegmentId(), descriptor1_1);
    segmentMap.put(descriptor1_2.getSegmentId(), descriptor1_2);
    segmentMap.put(descriptor1_4.getSegmentId(), descriptor1_4);
    segmentMap.put(descriptor1_5.getSegmentId(), descriptor1_5);
    segmentMap.put(descriptor2_3.getSegmentId(), descriptor2_3);
    segmentMap.put(descriptor2_4.getSegmentId(), descriptor2_4);
    segmentMap.put(descriptor2_5.getSegmentId(), descriptor2_5);
    segmentMap.put(descriptor2_6.getSegmentId(), descriptor2_6);
    segmentMap.put(descriptor2_7.getSegmentId(), descriptor2_7);
    segmentMap.put(descriptor3_5.getSegmentId(), descriptor3_5);
    segmentMap.put(descriptor3_6.getSegmentId(), descriptor3_6);
    segmentMap.put(descriptor3_7.getSegmentId(), descriptor3_7);
    List<StringDoublePair> containerList0 = new ArrayList<>();
    containerList0.add(new StringDoublePair(descriptor1_1.getSegmentId(), 1d));
    containerList0.add(new StringDoublePair(descriptor2_3.getSegmentId(), 1d));
    containerList0.add(new StringDoublePair(descriptor2_5.getSegmentId(), 1d));
    containerList0.add(new StringDoublePair(descriptor3_5.getSegmentId(), 1d));
    List<StringDoublePair> containerList1 = new ArrayList<>();
    containerList1.add(new StringDoublePair(descriptor1_2.getSegmentId(), 1d));
    containerList1.add(new StringDoublePair(descriptor2_3.getSegmentId(), 0.5d));
    containerList1.add(new StringDoublePair(descriptor2_6.getSegmentId(), 1d));
    containerList1.add(new StringDoublePair(descriptor3_5.getSegmentId(), 1d));
    containerList1.add(new StringDoublePair(descriptor3_6.getSegmentId(), 1d));
    containerList1.add(new StringDoublePair(descriptor3_6.getSegmentId(), 1d));
    List<StringDoublePair> containerList2 = new ArrayList<>();
    containerList2.add(new StringDoublePair(descriptor1_4.getSegmentId(), 1d));
    containerList2.add(new StringDoublePair(descriptor1_5.getSegmentId(), 0.5d));
    containerList2.add(new StringDoublePair(descriptor2_3.getSegmentId(), 0.5d));
    containerList2.add(new StringDoublePair(descriptor2_3.getSegmentId(), 0.5d));
    containerList2.add(new StringDoublePair(descriptor2_7.getSegmentId(), 1d));
    containerList2.add(new StringDoublePair(descriptor3_7.getSegmentId(), 1d));
    containerResults.add(0, containerList0);
    containerResults.add(1, containerList1);
    containerResults.add(2, containerList2);
    List<TemporalObject> expectedResults = new ArrayList<>();
    List<String> segments3 = new ArrayList<>();
    segments3.add(descriptor3_5.getSegmentId());
    segments3.add(descriptor3_6.getSegmentId());
    segments3.add(descriptor3_7.getSegmentId());
    TemporalObject t1 = new TemporalObject(segments3, descriptor3_5.getObjectId(), 1f);
    expectedResults.add(t1);
    List<String> segments2 = new ArrayList<>();
    segments2.add(descriptor2_3.getSegmentId());
    segments2.add(descriptor2_5.getSegmentId());
    segments2.add(descriptor2_6.getSegmentId());
    segments2.add(descriptor2_7.getSegmentId());
    TemporalObject t2 = new TemporalObject(segments2, descriptor2_3.getObjectId(), 1f);
    expectedResults.add(t2);
    List<String> segments1 = new ArrayList<>();
    segments1.add(descriptor1_1.getSegmentId());
    segments1.add(descriptor1_2.getSegmentId());
    segments1.add(descriptor1_4.getSegmentId());
    segments1.add(descriptor1_5.getSegmentId());
    TemporalObject t3 = new TemporalObject(segments1, descriptor1_1.getObjectId(), 2f / 3f);
    expectedResults.add(t3);
    this.segmentMap = segmentMap;
    this.containerResults = containerResults;
    this.maxLength = 30f;
    this.expectedResults = expectedResults;
    List<Float> timeDistances = new ArrayList<>();
    timeDistances.add(0f);
    timeDistances.add(0f);
    this.timeDistances = timeDistances;
}

Example 3 with StringDoublePair

use of org.vitrivr.cineast.core.data.StringDoublePair in project cineast by vitrivr.

the class TemporalTestCases method buildTestCase2.

public void buildTestCase2() {
    Map<String, MediaSegmentDescriptor> segmentMap = new HashMap<>();
    List<List<StringDoublePair>> containerResults = new ArrayList<>();
    segmentMap.put(descriptor1_1.getSegmentId(), descriptor1_1);
    segmentMap.put(descriptor1_2.getSegmentId(), descriptor1_2);
    segmentMap.put(descriptor1_3.getSegmentId(), descriptor1_3);
    List<StringDoublePair> containerList0 = new ArrayList<>();
    containerList0.add(new StringDoublePair(descriptor1_1.getSegmentId(), 1d));
    List<StringDoublePair> containerList1 = new ArrayList<>();
    containerList1.add(new StringDoublePair(descriptor1_2.getSegmentId(), 1d));
    containerList1.add(new StringDoublePair(descriptor1_3.getSegmentId(), 0.5d));
    containerResults.add(0, containerList0);
    containerResults.add(1, containerList1);
    List<TemporalObject> expectedResults = new ArrayList<>();
    List<String> segments1 = new ArrayList<>();
    segments1.add(descriptor1_1.getSegmentId());
    segments1.add(descriptor1_2.getSegmentId());
    segments1.add(descriptor1_3.getSegmentId());
    TemporalObject t1 = new TemporalObject(segments1, descriptor1_1.getObjectId(), 1f);
    expectedResults.add(t1);
    this.segmentMap = segmentMap;
    this.containerResults = containerResults;
    this.maxLength = 20f;
    this.expectedResults = expectedResults;
    List<Float> timeDistances = new ArrayList<>();
    timeDistances.add(0f);
    this.timeDistances = timeDistances;
}

Example 4 with StringDoublePair

use of org.vitrivr.cineast.core.data.StringDoublePair in project cineast by vitrivr.

the class TemporalTestCases method buildTestCase3.

public void buildTestCase3() {
    Map<String, MediaSegmentDescriptor> segmentMap = new HashMap<>();
    List<List<StringDoublePair>> containerResults = new ArrayList<>();
    segmentMap.put(descriptor1_1.getSegmentId(), descriptor1_1);
    segmentMap.put(descriptor1_2.getSegmentId(), descriptor1_2);
    segmentMap.put(descriptor1_3.getSegmentId(), descriptor1_3);
    segmentMap.put(descriptor1_4.getSegmentId(), descriptor1_4);
    segmentMap.put(descriptor1_5.getSegmentId(), descriptor1_5);
    segmentMap.put(descriptor1_6.getSegmentId(), descriptor1_6);
    segmentMap.put(descriptor1_7.getSegmentId(), descriptor1_7);
    List<StringDoublePair> containerList0 = new ArrayList<>();
    containerList0.add(new StringDoublePair(descriptor1_1.getSegmentId(), 1d));
    List<StringDoublePair> containerList1 = new ArrayList<>();
    containerList1.add(new StringDoublePair(descriptor1_2.getSegmentId(), 1d));
    containerList1.add(new StringDoublePair(descriptor1_3.getSegmentId(), 0.5d));
    containerList1.add(new StringDoublePair(descriptor1_7.getSegmentId(), 2d));
    List<StringDoublePair> containerList2 = new ArrayList<>();
    containerList2.add(new StringDoublePair(descriptor1_3.getSegmentId(), 0.5d));
    containerList2.add(new StringDoublePair(descriptor1_3.getSegmentId(), 0.5d));
    containerList2.add(new StringDoublePair(descriptor1_4.getSegmentId(), 1d));
    containerList2.add(new StringDoublePair(descriptor1_5.getSegmentId(), 0.5d));
    containerList2.add(new StringDoublePair(descriptor1_5.getSegmentId(), 1d));
    containerResults.add(0, containerList0);
    containerResults.add(1, containerList1);
    containerResults.add(2, containerList2);
    List<TemporalObject> expectedResults = new ArrayList<>();
    List<String> segments1 = new ArrayList<>();
    segments1.add(descriptor1_1.getSegmentId());
    segments1.add(descriptor1_2.getSegmentId());
    segments1.add(descriptor1_3.getSegmentId());
    segments1.add(descriptor1_4.getSegmentId());
    segments1.add(descriptor1_5.getSegmentId());
    segments1.add(descriptor1_7.getSegmentId());
    /* optimal result is 1: (1,1), 2: (2,1) 3: (3, 0.5) , because 4 is not a valid result */
    TemporalObject t1 = new TemporalObject(segments1, descriptor1_1.getObjectId(), 2.5 / 3f);
    expectedResults.add(t1);
    this.segmentMap = segmentMap;
    this.containerResults = containerResults;
    this.maxLength = 30f;
    this.expectedResults = expectedResults;
    List<Float> timeDistances = new ArrayList<>();
    timeDistances.add(0f);
    timeDistances.add(0f);
    this.timeDistances = timeDistances;
}

Example 5 with StringDoublePair

use of org.vitrivr.cineast.core.data.StringDoublePair in project cineast by vitrivr.

the class QueryUtil method retrieveCategory.

public static List<StringDoublePair> retrieveCategory(ContinuousRetrievalLogic continuousRetrievalLogic, List<Pair<AbstractQueryTermContainer, ReadableQueryConfig>> queryContainers, String category) {
    TObjectDoubleHashMap<String> scoreBySegmentId = new TObjectDoubleHashMap<>();
    for (Pair<AbstractQueryTermContainer, ReadableQueryConfig> pair : queryContainers) {
        if (pair == null) {
            continue;
        }
        AbstractQueryTermContainer qc = pair.first;
        ReadableQueryConfig qconf = pair.second;
        float weight = MathHelper.limit(qc.getWeight(), -1f, 1f);
        retrieveAndWeight(continuousRetrievalLogic, category, scoreBySegmentId, qc, qconf, weight);
    }
    final List<StringDoublePair> list = new ArrayList<>(scoreBySegmentId.size());
    scoreBySegmentId.forEachEntry((segmentId, score) -> {
        if (score > 0) {
            list.add(new StringDoublePair(segmentId, score));
        }
        return true;
    });
    list.sort(StringDoublePair.COMPARATOR);
    // FIXME: Using an arbitrary query config to limit results is prone to errors
    final int MAX_RESULTS = queryContainers.get(0).second.getMaxResults().orElse(Config.sharedConfig().getRetriever().getMaxResults());
    List<StringDoublePair> resultList = list;
    if (list.size() > MAX_RESULTS) {
        resultList = resultList.subList(0, MAX_RESULTS);
    }
    return resultList;
}