Search in sources :

Example 1 with AbstractSelectFilter

use of com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter in project conquery by bakdata.

the class FilterResolutionExactTest method execute.

@Override
public void execute(StandaloneSupport conquery) throws Exception {
    // read test sepcification
    String testJson = In.resource("/tests/query/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY.test.json").withUTF8().readAll();
    DatasetId dataset = conquery.getDataset().getId();
    ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson);
    ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
    CSVConfig csvConf = conquery.getConfig().getCsv();
    test.importRequiredData(conquery);
    FilterSearch.updateSearch(conquery.getNamespace().getNamespaces(), Collections.singleton(conquery.getNamespace().getDataset()), conquery.getDatasetsProcessor().getJobManager(), csvConf);
    conquery.waitUntilWorkDone();
    Concept<?> concept = conquery.getNamespace().getStorage().getAllConcepts().iterator().next();
    Connector connector = concept.getConnectors().iterator().next();
    AbstractSelectFilter<?> filter = (AbstractSelectFilter<?>) connector.getFilters().iterator().next();
    // Copy search csv from resources to tmp folder.
    final Path tmpCSv = Files.createTempFile("conquery_search", "csv");
    Files.write(tmpCSv, String.join(csvConf.getLineSeparator(), lines).getBytes(), StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
    filter.setSearchType(FilterSearch.FilterSearchType.EXACT);
    filter.setTemplate(new FilterTemplate(tmpCSv.toString(), Arrays.asList("HEADER"), "HEADER", "", ""));
    FilterSearch.createSourceSearch(filter, csvConf);
    assertThat(filter.getSourceSearch()).isNotNull();
    ConceptsProcessor processor = new ConceptsProcessor(conquery.getNamespace().getNamespaces());
    // from csv
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("a", "aaa", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("a", "aaa");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
    // from column values
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("f", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("f");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
}
Also used : Path(java.nio.file.Path) Connector(com.bakdata.conquery.models.datasets.concepts.Connector) AbstractSelectFilter(com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter) ConqueryTestSpec(com.bakdata.conquery.integration.json.ConqueryTestSpec) ConceptsProcessor(com.bakdata.conquery.resources.api.ConceptsProcessor) CSVConfig(com.bakdata.conquery.models.config.CSVConfig) DatasetId(com.bakdata.conquery.models.identifiable.ids.specific.DatasetId) FilterTemplate(com.bakdata.conquery.apiv1.FilterTemplate) ResolvedConceptsResult(com.bakdata.conquery.resources.api.ConceptsProcessor.ResolvedConceptsResult)

Example 2 with AbstractSelectFilter

use of com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter in project conquery by bakdata.

the class FilterResolutionPrefixTest method execute.

@Override
public void execute(StandaloneSupport conquery) throws Exception {
    // read test specification
    String testJson = In.resource("/tests/query/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY.test.json").withUTF8().readAll();
    DatasetId dataset = conquery.getDataset().getId();
    ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson);
    ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
    test.importRequiredData(conquery);
    CSVConfig csvConf = conquery.getConfig().getCsv();
    FilterSearch.updateSearch(conquery.getNamespace().getNamespaces(), Collections.singleton(conquery.getNamespace().getDataset()), conquery.getDatasetsProcessor().getJobManager(), csvConf);
    conquery.waitUntilWorkDone();
    Concept<?> concept = conquery.getNamespace().getStorage().getAllConcepts().iterator().next();
    Connector connector = concept.getConnectors().iterator().next();
    AbstractSelectFilter<?> filter = (AbstractSelectFilter<?>) connector.getFilters().iterator().next();
    // Copy search csv from resources to tmp folder.
    final Path tmpCSv = Files.createTempFile("conquery_search", "csv");
    Files.write(tmpCSv, String.join(csvConf.getLineSeparator(), lines).getBytes(), StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
    filter.setSearchType(FilterSearch.FilterSearchType.PREFIX);
    filter.setTemplate(new FilterTemplate(tmpCSv.toString(), Arrays.asList("HEADER"), "HEADER", "", ""));
    FilterSearch.createSourceSearch(filter, csvConf);
    assertThat(filter.getSourceSearch()).isNotNull();
    ConceptsProcessor processor = new ConceptsProcessor(conquery.getNamespace().getNamespaces());
    // from csv
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("a", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("a", "aaa", "aab");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
    // from column values
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("f", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("f");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
}
Also used : Path(java.nio.file.Path) Connector(com.bakdata.conquery.models.datasets.concepts.Connector) AbstractSelectFilter(com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter) ConqueryTestSpec(com.bakdata.conquery.integration.json.ConqueryTestSpec) ConceptsProcessor(com.bakdata.conquery.resources.api.ConceptsProcessor) CSVConfig(com.bakdata.conquery.models.config.CSVConfig) DatasetId(com.bakdata.conquery.models.identifiable.ids.specific.DatasetId) FilterTemplate(com.bakdata.conquery.apiv1.FilterTemplate) ResolvedConceptsResult(com.bakdata.conquery.resources.api.ConceptsProcessor.ResolvedConceptsResult)

Example 3 with AbstractSelectFilter

use of com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter in project conquery by bakdata.

the class ConceptsProcessor method resolveFilterValues.

/**
 * Search for all search terms at once, with stricter scoring.
 * The user will upload a file and expect only well-corresponding resolutions.
 */
public ResolvedConceptsResult resolveFilterValues(AbstractSelectFilter<?> filter, List<String> searchTerms) {
    // search in the full text engine
    Set<String> searchResult = createSourceSearchResult(filter.getSourceSearch(), searchTerms, OptionalInt.empty(), filter.getSearchType()::score).stream().map(FEValue::getValue).collect(Collectors.toSet());
    Set<String> openSearchTerms = new HashSet<>(searchTerms);
    openSearchTerms.removeAll(searchResult);
    // Iterate over all unresolved search terms. Gather all that match labels into searchResults. Keep the unresolvable ones.
    for (Iterator<String> it = openSearchTerms.iterator(); it.hasNext(); ) {
        String searchTerm = it.next();
        // Test if any of the values occurs directly in the filter's values or their labels (for when we don't have a provided file).
        if (filter.getValues().contains(searchTerm)) {
            searchResult.add(searchTerm);
            it.remove();
        } else {
            String matchingValue = filter.getLabels().inverse().get(searchTerm);
            if (matchingValue != null) {
                searchResult.add(matchingValue);
                it.remove();
            }
        }
    }
    return new ResolvedConceptsResult(null, new ResolvedFilterResult(filter.getConnector().getId(), filter.getId(), searchResult.stream().map(v -> new FEValue(filter.getLabelFor(v), v)).collect(Collectors.toList())), new ArrayList<>(openSearchTerms));
}
Also used : ConceptTreeChild(com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeChild) ConnectorId(com.bakdata.conquery.models.identifiable.ids.specific.ConnectorId) LoadingCache(com.google.common.cache.LoadingCache) RequiredArgsConstructor(lombok.RequiredArgsConstructor) Pair(org.apache.commons.lang3.tuple.Pair) DatasetId(com.bakdata.conquery.models.identifiable.ids.specific.DatasetId) ToString(lombok.ToString) TreeConcept(com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept) QuickSearch(com.bakdata.conquery.util.search.QuickSearch) Collection(java.util.Collection) Concept(com.bakdata.conquery.models.datasets.concepts.Concept) Set(java.util.Set) ConceptConfigurationException(com.bakdata.conquery.models.exceptions.ConceptConfigurationException) Collectors(java.util.stream.Collectors) FEList(com.bakdata.conquery.apiv1.frontend.FEList) Dataset(com.bakdata.conquery.models.datasets.Dataset) FilterId(com.bakdata.conquery.models.identifiable.ids.specific.FilterId) CacheLoader(com.google.common.cache.CacheLoader) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) Stream(java.util.stream.Stream) Ability(com.bakdata.conquery.models.auth.permissions.Ability) Optional(java.util.Optional) CacheBuilder(com.google.common.cache.CacheBuilder) Setter(lombok.Setter) FilterSearchItem(com.bakdata.conquery.apiv1.FilterSearchItem) Getter(lombok.Getter) Subject(com.bakdata.conquery.models.auth.entities.Subject) OptionalInt(java.util.OptionalInt) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Strings(com.google.common.base.Strings) NamespaceStorage(com.bakdata.conquery.io.storage.NamespaceStorage) FERoot(com.bakdata.conquery.apiv1.frontend.FERoot) LinkedList(java.util.LinkedList) SearchScorer(com.bakdata.conquery.util.search.SearchScorer) Iterator(java.util.Iterator) ConceptElementId(com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId) FEValue(com.bakdata.conquery.apiv1.frontend.FEValue) IdLabel(com.bakdata.conquery.apiv1.IdLabel) CalculatedValue(com.bakdata.conquery.util.CalculatedValue) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) AbstractSelectFilter(com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter) Data(lombok.Data) FilterSearch(com.bakdata.conquery.apiv1.FilterSearch) FrontEndConceptBuilder(com.bakdata.conquery.models.datasets.concepts.FrontEndConceptBuilder) DatasetRegistry(com.bakdata.conquery.models.worker.DatasetRegistry) Preconditions(com.google.common.base.Preconditions) Comparator(java.util.Comparator) AllArgsConstructor(lombok.AllArgsConstructor) Collections(java.util.Collections) ToString(lombok.ToString) FEValue(com.bakdata.conquery.apiv1.frontend.FEValue) HashSet(java.util.HashSet)

Example 4 with AbstractSelectFilter

use of com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter in project conquery by bakdata.

the class ConceptsProcessor method autocompleteTextFilter.

/**
 * Autocompletion for search terms. For values of {@link AbstractSelectFilter<?>}.
 * Is used by the serach cache to load missing items
 */
private static List<FEValue> autocompleteTextFilter(AbstractSelectFilter<?> filter, String text) {
    if (Strings.isNullOrEmpty(text)) {
        // If no text provided, we just list them
        // Filter might not have a source search (since none might be defined).
        // TODO unify these code paths, they are quite the mess, maybe also create source search for key-value also
        final Stream<FEValue> fromSearch = filter.getSourceSearch() == null ? Stream.empty() : filter.getSourceSearch().listItems().stream().map(item -> new FEValue(item.getLabel(), item.getValue(), item.getTemplateValues(), item.getOptionValue()));
        final Stream<FEValue> fromLabels = filter.getLabels().entrySet().stream().map(entry -> new FEValue(entry.getValue(), entry.getKey()));
        return Stream.concat(fromLabels, fromSearch).sorted().collect(Collectors.toList());
    }
    List<FEValue> result = new LinkedList<>();
    QuickSearch<FilterSearchItem> search = filter.getSourceSearch();
    if (search != null) {
        result = createSourceSearchResult(filter.getSourceSearch(), Collections.singletonList(text), OptionalInt.empty(), FilterSearch.FilterSearchType.CONTAINS::score);
    }
    String value = filter.getValueFor(text);
    if (value != null) {
        result.add(new FEValue(text, value));
    }
    return result;
}
Also used : ConceptTreeChild(com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeChild) ConnectorId(com.bakdata.conquery.models.identifiable.ids.specific.ConnectorId) LoadingCache(com.google.common.cache.LoadingCache) RequiredArgsConstructor(lombok.RequiredArgsConstructor) Pair(org.apache.commons.lang3.tuple.Pair) DatasetId(com.bakdata.conquery.models.identifiable.ids.specific.DatasetId) ToString(lombok.ToString) TreeConcept(com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept) QuickSearch(com.bakdata.conquery.util.search.QuickSearch) Collection(java.util.Collection) Concept(com.bakdata.conquery.models.datasets.concepts.Concept) Set(java.util.Set) ConceptConfigurationException(com.bakdata.conquery.models.exceptions.ConceptConfigurationException) Collectors(java.util.stream.Collectors) FEList(com.bakdata.conquery.apiv1.frontend.FEList) Dataset(com.bakdata.conquery.models.datasets.Dataset) FilterId(com.bakdata.conquery.models.identifiable.ids.specific.FilterId) CacheLoader(com.google.common.cache.CacheLoader) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) Stream(java.util.stream.Stream) Ability(com.bakdata.conquery.models.auth.permissions.Ability) Optional(java.util.Optional) CacheBuilder(com.google.common.cache.CacheBuilder) Setter(lombok.Setter) FilterSearchItem(com.bakdata.conquery.apiv1.FilterSearchItem) Getter(lombok.Getter) Subject(com.bakdata.conquery.models.auth.entities.Subject) OptionalInt(java.util.OptionalInt) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Strings(com.google.common.base.Strings) NamespaceStorage(com.bakdata.conquery.io.storage.NamespaceStorage) FERoot(com.bakdata.conquery.apiv1.frontend.FERoot) LinkedList(java.util.LinkedList) SearchScorer(com.bakdata.conquery.util.search.SearchScorer) Iterator(java.util.Iterator) ConceptElementId(com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId) FEValue(com.bakdata.conquery.apiv1.frontend.FEValue) IdLabel(com.bakdata.conquery.apiv1.IdLabel) CalculatedValue(com.bakdata.conquery.util.CalculatedValue) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) AbstractSelectFilter(com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter) Data(lombok.Data) FilterSearch(com.bakdata.conquery.apiv1.FilterSearch) FrontEndConceptBuilder(com.bakdata.conquery.models.datasets.concepts.FrontEndConceptBuilder) DatasetRegistry(com.bakdata.conquery.models.worker.DatasetRegistry) Preconditions(com.google.common.base.Preconditions) Comparator(java.util.Comparator) AllArgsConstructor(lombok.AllArgsConstructor) Collections(java.util.Collections) FilterSearchItem(com.bakdata.conquery.apiv1.FilterSearchItem) ToString(lombok.ToString) FEValue(com.bakdata.conquery.apiv1.frontend.FEValue) LinkedList(java.util.LinkedList)

Example 5 with AbstractSelectFilter

use of com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter in project conquery by bakdata.

the class FilterResolutionContainsTest method execute.

@Override
public void execute(StandaloneSupport conquery) throws Exception {
    // read test sepcification
    String testJson = In.resource("/tests/query/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY.test.json").withUTF8().readAll();
    DatasetId dataset = conquery.getDataset().getId();
    ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson);
    ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
    test.importRequiredData(conquery);
    CSVConfig csvConf = conquery.getConfig().getCsv();
    FilterSearch.updateSearch(conquery.getNamespace().getNamespaces(), Collections.singleton(conquery.getNamespace().getDataset()), conquery.getDatasetsProcessor().getJobManager(), csvConf);
    conquery.waitUntilWorkDone();
    Concept<?> concept = conquery.getNamespace().getStorage().getAllConcepts().iterator().next();
    Connector connector = concept.getConnectors().iterator().next();
    AbstractSelectFilter<?> filter = (AbstractSelectFilter<?>) connector.getFilters().iterator().next();
    // Copy search csv from resources to tmp folder.
    final Path tmpCSv = Files.createTempFile("conquery_search", ".csv");
    Out.file(tmpCSv.toFile()).withUTF8().writeLines(lines);
    Files.write(tmpCSv, String.join(csvConf.getLineSeparator(), lines).getBytes(), StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
    filter.setSearchType(FilterSearch.FilterSearchType.CONTAINS);
    filter.setTemplate(new FilterTemplate(tmpCSv.toString(), Arrays.asList("HEADER"), "HEADER", "", ""));
    FilterSearch.createSourceSearch(filter, csvConf);
    assertThat(filter.getSourceSearch()).isNotNull();
    ConceptsProcessor processor = new ConceptsProcessor(conquery.getNamespace().getNamespaces());
    // from csv
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("a", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("a", "aaa", "aab", "baaa");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
    // from column values
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("f", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("f");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
}
Also used : Path(java.nio.file.Path) Connector(com.bakdata.conquery.models.datasets.concepts.Connector) AbstractSelectFilter(com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter) ConqueryTestSpec(com.bakdata.conquery.integration.json.ConqueryTestSpec) ConceptsProcessor(com.bakdata.conquery.resources.api.ConceptsProcessor) CSVConfig(com.bakdata.conquery.models.config.CSVConfig) DatasetId(com.bakdata.conquery.models.identifiable.ids.specific.DatasetId) FilterTemplate(com.bakdata.conquery.apiv1.FilterTemplate) ResolvedConceptsResult(com.bakdata.conquery.resources.api.ConceptsProcessor.ResolvedConceptsResult)

Aggregations

AbstractSelectFilter (com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter)5 DatasetId (com.bakdata.conquery.models.identifiable.ids.specific.DatasetId)5 FilterTemplate (com.bakdata.conquery.apiv1.FilterTemplate)3 ConqueryTestSpec (com.bakdata.conquery.integration.json.ConqueryTestSpec)3 CSVConfig (com.bakdata.conquery.models.config.CSVConfig)3 Connector (com.bakdata.conquery.models.datasets.concepts.Connector)3 ConceptsProcessor (com.bakdata.conquery.resources.api.ConceptsProcessor)3 ResolvedConceptsResult (com.bakdata.conquery.resources.api.ConceptsProcessor.ResolvedConceptsResult)3 Path (java.nio.file.Path)3 FilterSearch (com.bakdata.conquery.apiv1.FilterSearch)2 FilterSearchItem (com.bakdata.conquery.apiv1.FilterSearchItem)2 IdLabel (com.bakdata.conquery.apiv1.IdLabel)2 FEList (com.bakdata.conquery.apiv1.frontend.FEList)2 FERoot (com.bakdata.conquery.apiv1.frontend.FERoot)2 FEValue (com.bakdata.conquery.apiv1.frontend.FEValue)2 NamespaceStorage (com.bakdata.conquery.io.storage.NamespaceStorage)2 Subject (com.bakdata.conquery.models.auth.entities.Subject)2 Ability (com.bakdata.conquery.models.auth.permissions.Ability)2 Dataset (com.bakdata.conquery.models.datasets.Dataset)2 Concept (com.bakdata.conquery.models.datasets.concepts.Concept)2