Search in sources :

Example 1 with MediaObjectUtil

use of org.vitrivr.cineast.api.grpc.util.MediaObjectUtil in project cineast by vitrivr.

the class CineastQueryService method getSimilar.

// TODO This has enormous code duplication with the TemporalQueryMessageHandler
@Override
public void getSimilar(CineastGrpc.TemporalQuery query, StreamObserver<CineastGrpc.QueryResult> responseObserver) {
    StopWatch watch = StopWatch.createStarted();
    MediaSegmentReader mediaSegmentReader = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    MediaObjectReader mediaObjectReader = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    MediaSegmentMetadataReader segmentMetadataReader = new MediaSegmentMetadataReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    MediaObjectMetadataReader objectMetadataReader = new MediaObjectMetadataReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    Set<String> sentSegmentIds = new HashSet<>(), sentObjectIds = new HashSet<>();
    CineastGrpc.QueryConfig config = query.getQueryList().get(0).getConfig();
    ReadableQueryConfig rqconf = QueryContainerUtil.queryConfig(config);
    QueryConfig qconf = new QueryConfig(rqconf);
    /* Prepare QueryConfig (so as to obtain a QueryId). */
    final String uuid = qconf.getQueryId().toString();
    final int max = qconf.getMaxResults().orElse(Config.sharedConfig().getRetriever().getMaxResults());
    qconf.setMaxResults(max);
    final int resultsPerModule = qconf.getRawResultsPerModule() == -1 ? Config.sharedConfig().getRetriever().getMaxResultsPerModule() : qconf.getResultsPerModule();
    qconf.setResultsPerModule(resultsPerModule);
    List<Thread> metadataRetrievalThreads = new ArrayList<>();
    /* We iterate over all components independently, because they have a temporal context.*/
    for (int containerIdx = 0; containerIdx < query.getQueryCount(); containerIdx++) {
        List<QueryStage> stages = QueryContainerUtil.query(query.getQueryList().get(containerIdx));
        /* We make a new stagedQueryConfig per stage because the relevant segments will differ for each stage. This also resets the filter (relevant ids in the config)*/
        QueryConfig stageQConf = QueryConfig.clone(qconf);
        /* For the first stage, there will be no relevant segments when querying. This is ok because the retrieval engine handles this appropriately */
        HashSet<String> relevantSegments = new HashSet<>();
        /* Store for each queryterm per category all results to be sent at a later time */
        List<Map<String, List<StringDoublePair>>> cache = new ArrayList<>();
        /* For the terms of a stage, ordering matters. The assumption is that each term is used as a filter for its successor */
        for (int stageIndex = 0; stageIndex < stages.size(); stageIndex++) {
            /* Initalize stage with this hashmap */
            cache.add(stageIndex, new HashMap<>());
            QueryStage stage = stages.get(stageIndex);
            List<Thread> qtThreads = new ArrayList<>();
            /* We now iterate over all QueryTerms for this stage, simply adding their results to the list of relevant segments for the next querystage.
         * The list is only updated once we've iterated over all terms
         */
            for (int i = 0; i < stage.getQueryTerms().size(); i++) {
                QueryTerm qt = stage.getQueryTerms().get(i);
                final int finalContainerIdx = containerIdx;
                final int finalStageIndex = stageIndex;
                Thread qtRetrievalThread = new Thread(() -> {
                    /* Prepare QueryTerm and perform sanity-checks */
                    if (qt == null) {
                        /* In rare instances, it is possible to have null as query stage. If this happens to you, please report this to the developers so we can try to fix it. */
                        LOGGER.warn("QueryTerm was null for stage {}", stage);
                        return;
                    }
                    AbstractQueryTermContainer qc = qt.getContainer();
                    if (qc == null) {
                        LOGGER.warn("Likely an empty query, as it could not be converted to a query container. Ignoring it");
                        return;
                    }
                    List<Thread> categoryThreads = new ArrayList<>();
                    /* For each category of a specific queryterm, we actually go and retrieve. Be aware that we do not change the relevant ids after this call */
                    for (String category : qt.getCategories()) {
                        /* Merge partial results with score-map */
                        List<SegmentScoreElement> scores = continuousRetrievalLogic.retrieve(qc, category, stageQConf);
                        /* Transform raw results into list of StringDoublePairs (segmentId -> score) */
                        final List<StringDoublePair> results = scores.stream().map(elem -> new StringDoublePair(elem.getSegmentId(), elem.getScore())).filter(p -> p.value > 0d).sorted(StringDoublePair.COMPARATOR).limit(max).collect(Collectors.toList());
                        if (results.isEmpty()) {
                            LOGGER.warn("No results found for category {} and qt {} in stage with id {}. Full compoment: {}", category, qt, finalContainerIdx, stage);
                        }
                        if (cache.get(finalStageIndex).containsKey(category)) {
                            LOGGER.error("Category {} was used twice in stage {}. This erases the results of the previous category... ", category, finalStageIndex);
                        }
                        cache.get(finalStageIndex).put(category, results);
                        results.forEach(res -> relevantSegments.add(res.key));
                        LOGGER.trace("Category {} at stage {} executed @ {} ms", category, finalStageIndex, watch.getTime(TimeUnit.MILLISECONDS));
                        /* If this is the last stage, we can send relevant results per category back to the UI.
               * Otherwise, we cannot since we might send results to the UI which would be filtered at a later stage
               */
                        if (finalStageIndex == stages.size() - 1) {
                            /* Finalize and submit per-container results */
                            responseObserver.onNext(QueryContainerUtil.queryResult(QueryContainerUtil.similarityQueryResult(qt.getQueryConfig().getQueryId().toString(), category, results)));
                            List<String> segmentIds = results.stream().map(x -> x.key).filter(x -> !sentSegmentIds.contains(x)).collect(Collectors.toList());
                            if (segmentIds.isEmpty()) {
                                continue;
                            }
                            Map<String, MediaSegmentDescriptor> segments = mediaSegmentReader.lookUpSegments(segmentIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaSegmentQueryResult.newBuilder().addAllSegments(segments.values().stream().map(MediaSegmentUtil::fromMediaSegmentDescriptor).collect(Collectors.toList())).build()));
                            List<MediaSegmentMetadataDescriptor> segmentMetaData = segmentMetadataReader.lookupMultimediaMetadata(segmentIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaSegmentMetaDataQueryResult.newBuilder().addAllSegmentMetaData(segmentMetaData.stream().map(QueryContainerUtil::mediaSegmentMetaData).collect(Collectors.toList())).build()));
                            sentSegmentIds.addAll(segmentIds);
                            List<String> objectIds = segments.values().stream().map(MediaSegmentDescriptor::getObjectId).filter(x -> !sentObjectIds.contains(x)).collect(Collectors.toList());
                            if (objectIds.isEmpty()) {
                                continue;
                            }
                            Map<String, MediaObjectDescriptor> objects = mediaObjectReader.lookUpObjects(objectIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaObjectQueryResult.newBuilder().addAllObjects(objects.values().stream().map(MediaObjectUtil::fromMediaObjectDescriptor).collect(Collectors.toList())).build()));
                            List<MediaObjectMetadataDescriptor> objectMetaData = objectMetadataReader.lookupMultimediaMetadata(objectIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaObjectMetaDataQueryResult.newBuilder().addAllObjectMetaData(objectMetaData.stream().map(QueryContainerUtil::mediaObjectMetaData).collect(Collectors.toList())).build()));
                            sentObjectIds.addAll(objectIds);
                        }
                    }
                /* We're done for this querycontainer */
                });
                // TODO Better name
                qtRetrievalThread.setName("qt-stage" + stageIndex + "-" + qt.getCategories());
                qtThreads.add(qtRetrievalThread);
                qtRetrievalThread.start();
            }
            for (Thread thread : qtThreads) {
                try {
                    thread.join();
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
            /* After we are done with a stage, we add all relevant segments to the config for the next stage. */
            if (relevantSegments.size() == 0) {
                LOGGER.warn("No relevant segments anymore, aborting staged querying");
                /* Clear relevant segments (there are none) */
                stageQConf.setRelevantSegmentIds(relevantSegments);
                break;
            }
            stageQConf.setRelevantSegmentIds(relevantSegments);
            relevantSegments.clear();
        }
        /* At this point, we have iterated over all stages. Now, we need to go back for all stages and send the results for the relevant ids. */
        for (int stageIndex = 0; stageIndex < stages.size() - 1; stageIndex++) {
            cache.get(stageIndex).forEach((category, results) -> {
                results.removeIf(pair -> !stageQConf.getRelevantSegmentIds().contains(pair.key));
                responseObserver.onNext(QueryContainerUtil.queryResult(QueryContainerUtil.similarityQueryResult(// TODO This assumes that all queries in a temporalquery have the same uuid
                uuid, category, results)));
            });
        }
    /* There should be no carry-over from this block since temporal queries are executed independently */
    }
    /* At this point, all StagedQueries have been executed for this TemporalQuery.
     * Since results have always been sent for the final stage or, when appropriate, in intermediate steps, there's nothing left to do.
     */
    responseObserver.onCompleted();
    mediaSegmentReader.close();
    mediaObjectReader.close();
    segmentMetadataReader.close();
    watch.stop();
    LOGGER.debug("Query executed in {} ms", watch.getTime(TimeUnit.MILLISECONDS));
}
Also used : MediaObjectUtil(org.vitrivr.cineast.api.grpc.util.MediaObjectUtil) AbstractQueryTermContainer(org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer) HashMap(java.util.HashMap) MediaSegmentUtil(org.vitrivr.cineast.api.grpc.util.MediaSegmentUtil) QueryStage(org.vitrivr.cineast.api.grpc.data.QueryStage) MediaObjectMetadataDescriptor(org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor) MediaObjectDescriptor(org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) StreamObserver(io.grpc.stub.StreamObserver) MediaObjectReader(org.vitrivr.cineast.core.db.dao.reader.MediaObjectReader) QueryContainerUtil(org.vitrivr.cineast.api.grpc.util.QueryContainerUtil) Map(java.util.Map) MediaSegmentDescriptor(org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor) ContinuousRetrievalLogic(org.vitrivr.cineast.standalone.util.ContinuousRetrievalLogic) MediaSegmentReader(org.vitrivr.cineast.core.db.dao.reader.MediaSegmentReader) MediaObjectMetadataReader(org.vitrivr.cineast.core.db.dao.reader.MediaObjectMetadataReader) QueryConfig(org.vitrivr.cineast.core.config.QueryConfig) Set(java.util.Set) StopWatch(org.apache.commons.lang3.time.StopWatch) ReadableQueryConfig(org.vitrivr.cineast.core.config.ReadableQueryConfig) StringDoublePair(org.vitrivr.cineast.core.data.StringDoublePair) Collectors(java.util.stream.Collectors) TimeUnit(java.util.concurrent.TimeUnit) QueryUtil(org.vitrivr.cineast.api.util.QueryUtil) List(java.util.List) Logger(org.apache.logging.log4j.Logger) SegmentScoreElement(org.vitrivr.cineast.core.data.score.SegmentScoreElement) MediaSegmentMetadataDescriptor(org.vitrivr.cineast.core.data.entities.MediaSegmentMetadataDescriptor) MediaSegmentMetadataReader(org.vitrivr.cineast.core.db.dao.reader.MediaSegmentMetadataReader) LogManager(org.apache.logging.log4j.LogManager) QueryTerm(org.vitrivr.cineast.api.grpc.data.QueryTerm) Config(org.vitrivr.cineast.standalone.config.Config) MediaObjectDescriptor(org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor) AbstractQueryTermContainer(org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer) ArrayList(java.util.ArrayList) MediaObjectReader(org.vitrivr.cineast.core.db.dao.reader.MediaObjectReader) QueryTerm(org.vitrivr.cineast.api.grpc.data.QueryTerm) MediaObjectMetadataReader(org.vitrivr.cineast.core.db.dao.reader.MediaObjectMetadataReader) StringDoublePair(org.vitrivr.cineast.core.data.StringDoublePair) MediaSegmentMetadataDescriptor(org.vitrivr.cineast.core.data.entities.MediaSegmentMetadataDescriptor) QueryStage(org.vitrivr.cineast.api.grpc.data.QueryStage) MediaObjectMetadataDescriptor(org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor) HashSet(java.util.HashSet) MediaSegmentMetadataReader(org.vitrivr.cineast.core.db.dao.reader.MediaSegmentMetadataReader) QueryConfig(org.vitrivr.cineast.core.config.QueryConfig) ReadableQueryConfig(org.vitrivr.cineast.core.config.ReadableQueryConfig) StopWatch(org.apache.commons.lang3.time.StopWatch) ReadableQueryConfig(org.vitrivr.cineast.core.config.ReadableQueryConfig) MediaSegmentReader(org.vitrivr.cineast.core.db.dao.reader.MediaSegmentReader) SegmentScoreElement(org.vitrivr.cineast.core.data.score.SegmentScoreElement) MediaSegmentDescriptor(org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor) HashMap(java.util.HashMap) Map(java.util.Map)

Aggregations

StreamObserver (io.grpc.stub.StreamObserver)1 ArrayList (java.util.ArrayList)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Map (java.util.Map)1 Set (java.util.Set)1 TimeUnit (java.util.concurrent.TimeUnit)1 Collectors (java.util.stream.Collectors)1 StopWatch (org.apache.commons.lang3.time.StopWatch)1 LogManager (org.apache.logging.log4j.LogManager)1 Logger (org.apache.logging.log4j.Logger)1 QueryStage (org.vitrivr.cineast.api.grpc.data.QueryStage)1 QueryTerm (org.vitrivr.cineast.api.grpc.data.QueryTerm)1 MediaObjectUtil (org.vitrivr.cineast.api.grpc.util.MediaObjectUtil)1 MediaSegmentUtil (org.vitrivr.cineast.api.grpc.util.MediaSegmentUtil)1 QueryContainerUtil (org.vitrivr.cineast.api.grpc.util.QueryContainerUtil)1 QueryUtil (org.vitrivr.cineast.api.util.QueryUtil)1 QueryConfig (org.vitrivr.cineast.core.config.QueryConfig)1 ReadableQueryConfig (org.vitrivr.cineast.core.config.ReadableQueryConfig)1