Search in sources :

Example 1 with Connector

use of com.bakdata.conquery.models.datasets.concepts.Connector in project conquery by bakdata.

the class FilterResolutionExactTest method execute.

@Override
public void execute(StandaloneSupport conquery) throws Exception {
    // read test sepcification
    String testJson = In.resource("/tests/query/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY.test.json").withUTF8().readAll();
    DatasetId dataset = conquery.getDataset().getId();
    ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson);
    ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
    CSVConfig csvConf = conquery.getConfig().getCsv();
    test.importRequiredData(conquery);
    FilterSearch.updateSearch(conquery.getNamespace().getNamespaces(), Collections.singleton(conquery.getNamespace().getDataset()), conquery.getDatasetsProcessor().getJobManager(), csvConf);
    conquery.waitUntilWorkDone();
    Concept<?> concept = conquery.getNamespace().getStorage().getAllConcepts().iterator().next();
    Connector connector = concept.getConnectors().iterator().next();
    AbstractSelectFilter<?> filter = (AbstractSelectFilter<?>) connector.getFilters().iterator().next();
    // Copy search csv from resources to tmp folder.
    final Path tmpCSv = Files.createTempFile("conquery_search", "csv");
    Files.write(tmpCSv, String.join(csvConf.getLineSeparator(), lines).getBytes(), StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
    filter.setSearchType(FilterSearch.FilterSearchType.EXACT);
    filter.setTemplate(new FilterTemplate(tmpCSv.toString(), Arrays.asList("HEADER"), "HEADER", "", ""));
    FilterSearch.createSourceSearch(filter, csvConf);
    assertThat(filter.getSourceSearch()).isNotNull();
    ConceptsProcessor processor = new ConceptsProcessor(conquery.getNamespace().getNamespaces());
    // from csv
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("a", "aaa", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("a", "aaa");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
    // from column values
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("f", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("f");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
}
Also used : Path(java.nio.file.Path) Connector(com.bakdata.conquery.models.datasets.concepts.Connector) AbstractSelectFilter(com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter) ConqueryTestSpec(com.bakdata.conquery.integration.json.ConqueryTestSpec) ConceptsProcessor(com.bakdata.conquery.resources.api.ConceptsProcessor) CSVConfig(com.bakdata.conquery.models.config.CSVConfig) DatasetId(com.bakdata.conquery.models.identifiable.ids.specific.DatasetId) FilterTemplate(com.bakdata.conquery.apiv1.FilterTemplate) ResolvedConceptsResult(com.bakdata.conquery.resources.api.ConceptsProcessor.ResolvedConceptsResult)

Example 2 with Connector

use of com.bakdata.conquery.models.datasets.concepts.Connector in project conquery by bakdata.

the class FilterResolutionPrefixTest method execute.

@Override
public void execute(StandaloneSupport conquery) throws Exception {
    // read test specification
    String testJson = In.resource("/tests/query/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY.test.json").withUTF8().readAll();
    DatasetId dataset = conquery.getDataset().getId();
    ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson);
    ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
    test.importRequiredData(conquery);
    CSVConfig csvConf = conquery.getConfig().getCsv();
    FilterSearch.updateSearch(conquery.getNamespace().getNamespaces(), Collections.singleton(conquery.getNamespace().getDataset()), conquery.getDatasetsProcessor().getJobManager(), csvConf);
    conquery.waitUntilWorkDone();
    Concept<?> concept = conquery.getNamespace().getStorage().getAllConcepts().iterator().next();
    Connector connector = concept.getConnectors().iterator().next();
    AbstractSelectFilter<?> filter = (AbstractSelectFilter<?>) connector.getFilters().iterator().next();
    // Copy search csv from resources to tmp folder.
    final Path tmpCSv = Files.createTempFile("conquery_search", "csv");
    Files.write(tmpCSv, String.join(csvConf.getLineSeparator(), lines).getBytes(), StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
    filter.setSearchType(FilterSearch.FilterSearchType.PREFIX);
    filter.setTemplate(new FilterTemplate(tmpCSv.toString(), Arrays.asList("HEADER"), "HEADER", "", ""));
    FilterSearch.createSourceSearch(filter, csvConf);
    assertThat(filter.getSourceSearch()).isNotNull();
    ConceptsProcessor processor = new ConceptsProcessor(conquery.getNamespace().getNamespaces());
    // from csv
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("a", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("a", "aaa", "aab");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
    // from column values
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("f", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("f");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
}
Also used : Path(java.nio.file.Path) Connector(com.bakdata.conquery.models.datasets.concepts.Connector) AbstractSelectFilter(com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter) ConqueryTestSpec(com.bakdata.conquery.integration.json.ConqueryTestSpec) ConceptsProcessor(com.bakdata.conquery.resources.api.ConceptsProcessor) CSVConfig(com.bakdata.conquery.models.config.CSVConfig) DatasetId(com.bakdata.conquery.models.identifiable.ids.specific.DatasetId) FilterTemplate(com.bakdata.conquery.apiv1.FilterTemplate) ResolvedConceptsResult(com.bakdata.conquery.resources.api.ConceptsProcessor.ResolvedConceptsResult)

Example 3 with Connector

use of com.bakdata.conquery.models.datasets.concepts.Connector in project conquery by bakdata.

the class FilterTest method importConcepts.

private void importConcepts(StandaloneSupport support) throws JSONException, IOException {
    Dataset dataset = support.getDataset();
    concept = new TreeConcept();
    concept.setLabel("concept");
    concept.setValidator(support.getValidator());
    concept.setDataset(support.getDataset());
    rawConnector.put("name", "connector");
    rawConnector.put("table", "table");
    ((ObjectNode) rawConnector.get("filters")).put("name", "filter");
    connector = parseSubTree(support, rawConnector, ConceptTreeConnector.class, conn -> conn.setConcept(concept));
    concept.setConnectors(Collections.singletonList((ConceptTreeConnector) connector));
    support.getDatasetsProcessor().addConcept(dataset, concept);
}
Also used : JsonProperty(com.fasterxml.jackson.annotation.JsonProperty) Setter(lombok.Setter) Getter(lombok.Getter) RequiredData(com.bakdata.conquery.integration.common.RequiredData) ConceptTreeConnector(com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) Connector(com.bakdata.conquery.models.datasets.concepts.Connector) JsonIgnore(com.fasterxml.jackson.annotation.JsonIgnore) JSONException(com.bakdata.conquery.models.exceptions.JSONException) CQTable(com.bakdata.conquery.apiv1.query.concept.filter.CQTable) TreeConcept(com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept) Range(com.bakdata.conquery.models.common.Range) ResourceFile(com.bakdata.conquery.integration.common.ResourceFile) CPSType(com.bakdata.conquery.io.cps.CPSType) IOException(java.io.IOException) NotNull(javax.validation.constraints.NotNull) LoadingUtil(com.bakdata.conquery.integration.common.LoadingUtil) StandaloneSupport(com.bakdata.conquery.util.support.StandaloneSupport) AbstractQueryEngineTest(com.bakdata.conquery.integration.json.AbstractQueryEngineTest) Dataset(com.bakdata.conquery.models.datasets.Dataset) FilterValue(com.bakdata.conquery.apiv1.query.concept.filter.FilterValue) CQDateRestriction(com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction) ConceptQuery(com.bakdata.conquery.apiv1.query.ConceptQuery) Slf4j(lombok.extern.slf4j.Slf4j) LocalDate(java.time.LocalDate) Query(com.bakdata.conquery.apiv1.query.Query) Jackson(com.bakdata.conquery.io.jackson.Jackson) ConqueryTestSpec(com.bakdata.conquery.integration.json.ConqueryTestSpec) CQConcept(com.bakdata.conquery.apiv1.query.concept.specific.CQConcept) Collections(java.util.Collections) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) Dataset(com.bakdata.conquery.models.datasets.Dataset) TreeConcept(com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept) ConceptTreeConnector(com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector)

Example 4 with Connector

use of com.bakdata.conquery.models.datasets.concepts.Connector in project conquery by bakdata.

the class TableExportQuery method resolve.

@Override
public void resolve(QueryResolveContext context) {
    query.resolve(context);
    // First is dates
    AtomicInteger currentPosition = new AtomicInteger(1);
    positions = new HashMap<>();
    Map<SecondaryIdDescription, Integer> secondaryIdPositions = new HashMap<>();
    // SecondaryIds are pulled to the front and grouped over all tables
    tables.stream().map(cqUnfilteredTable -> cqUnfilteredTable.getTable().getTable().getColumns()).flatMap(Arrays::stream).map(Column::getSecondaryId).filter(Objects::nonNull).distinct().sorted(Comparator.comparing(SecondaryIdDescription::getLabel)).forEach(secondaryId -> secondaryIdPositions.put(secondaryId, currentPosition.getAndIncrement()));
    for (CQUnfilteredTable table : tables) {
        Connector connector = table.getTable();
        final Column validityDateColumn = findValidityDateColumn(connector, table.getDateColumn());
        if (validityDateColumn != null) {
            positions.putIfAbsent(validityDateColumn, 0);
        }
        // Set column positions, set SecondaryId positions to precomputed ones.
        for (Column column : connector.getTable().getColumns()) {
            positions.computeIfAbsent(column, col -> col.getSecondaryId() != null ? secondaryIdPositions.get(col.getSecondaryId()) : currentPosition.getAndIncrement());
        }
    }
    resultInfos = createResultInfos(currentPosition.get(), secondaryIdPositions, positions);
}
Also used : AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Connector(com.bakdata.conquery.models.datasets.concepts.Connector) CQUnfilteredTable(com.bakdata.conquery.apiv1.query.concept.filter.CQUnfilteredTable) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) HashMap(java.util.HashMap) Column(com.bakdata.conquery.models.datasets.Column) SecondaryIdDescription(com.bakdata.conquery.models.datasets.SecondaryIdDescription)

Example 5 with Connector

use of com.bakdata.conquery.models.datasets.concepts.Connector in project conquery by bakdata.

the class AdminDatasetProcessor method deleteTable.

/**
 * Deletes a table if it has no dependents or not forced to do so.
 */
public synchronized List<ConceptId> deleteTable(Table table, boolean force) {
    final Namespace namespace = datasetRegistry.get(table.getDataset().getId());
    final List<Concept<?>> dependentConcepts = namespace.getStorage().getAllConcepts().stream().flatMap(c -> c.getConnectors().stream()).filter(con -> con.getTable().equals(table)).map(Connector::getConcept).collect(Collectors.toList());
    if (force || dependentConcepts.isEmpty()) {
        for (Concept<?> concept : dependentConcepts) {
            deleteConcept(concept);
        }
        namespace.getStorage().getAllImports().stream().filter(imp -> imp.getTable().equals(table)).forEach(this::deleteImport);
        namespace.getStorage().removeTable(table.getId());
        namespace.sendToAll(new RemoveTable(table));
    }
    return dependentConcepts.stream().map(Concept::getId).collect(Collectors.toList());
}
Also used : Concept(com.bakdata.conquery.models.datasets.concepts.Concept) RemoveConcept(com.bakdata.conquery.models.messages.namespaces.specific.RemoveConcept) UpdateConcept(com.bakdata.conquery.models.messages.namespaces.specific.UpdateConcept) Arrays(java.util.Arrays) ConqueryConfig(com.bakdata.conquery.models.config.ConqueryConfig) SneakyThrows(lombok.SneakyThrows) RequiredArgsConstructor(lombok.RequiredArgsConstructor) EntityIdMap(com.bakdata.conquery.models.identifiable.mapping.EntityIdMap) Import(com.bakdata.conquery.models.datasets.Import) DictionaryId(com.bakdata.conquery.models.identifiable.ids.specific.DictionaryId) ConceptId(com.bakdata.conquery.models.identifiable.ids.specific.ConceptId) DatasetId(com.bakdata.conquery.models.identifiable.ids.specific.DatasetId) InternalOnly(com.bakdata.conquery.io.jackson.InternalOnly) RemoveImportJob(com.bakdata.conquery.models.messages.namespaces.specific.RemoveImportJob) NonNull(lombok.NonNull) Table(com.bakdata.conquery.models.datasets.Table) Concept(com.bakdata.conquery.models.datasets.concepts.Concept) RemoveConcept(com.bakdata.conquery.models.messages.namespaces.specific.RemoveConcept) Set(java.util.Set) Validator(javax.validation.Validator) Collectors(java.util.stream.Collectors) NotFoundException(javax.ws.rs.NotFoundException) RemoveSecondaryId(com.bakdata.conquery.models.messages.namespaces.specific.RemoveSecondaryId) Dataset(com.bakdata.conquery.models.datasets.Dataset) Objects(java.util.Objects) SecondaryIdDescription(com.bakdata.conquery.models.datasets.SecondaryIdDescription) JobManager(com.bakdata.conquery.models.jobs.JobManager) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) Response(javax.ws.rs.core.Response) WebApplicationException(javax.ws.rs.WebApplicationException) Jackson(com.bakdata.conquery.io.jackson.Jackson) MetaStorage(com.bakdata.conquery.io.storage.MetaStorage) Column(com.bakdata.conquery.models.datasets.Column) Namespace(com.bakdata.conquery.models.worker.Namespace) RemoveTable(com.bakdata.conquery.models.messages.namespaces.specific.RemoveTable) Getter(lombok.Getter) AddWorker(com.bakdata.conquery.models.messages.network.specific.AddWorker) UpdateTable(com.bakdata.conquery.models.messages.namespaces.specific.UpdateTable) Connector(com.bakdata.conquery.models.datasets.concepts.Connector) UpdateSecondaryId(com.bakdata.conquery.models.messages.namespaces.specific.UpdateSecondaryId) NamespaceStorage(com.bakdata.conquery.io.storage.NamespaceStorage) IdMutex(com.bakdata.conquery.models.identifiable.IdMutex) TableId(com.bakdata.conquery.models.identifiable.ids.specific.TableId) CsvParser(com.univocity.parsers.csv.CsvParser) UpdateMatchingStatsMessage(com.bakdata.conquery.models.messages.namespaces.specific.UpdateMatchingStatsMessage) Identifiable(com.bakdata.conquery.models.identifiable.Identifiable) SimpleJob(com.bakdata.conquery.models.jobs.SimpleJob) ForbiddenException(javax.ws.rs.ForbiddenException) ImportJob(com.bakdata.conquery.models.jobs.ImportJob) IOException(java.io.IOException) ValidatorHelper(com.bakdata.conquery.models.exceptions.ValidatorHelper) UpdateConcept(com.bakdata.conquery.models.messages.namespaces.specific.UpdateConcept) ShardNodeInformation(com.bakdata.conquery.models.worker.ShardNodeInformation) StructureNode(com.bakdata.conquery.models.datasets.concepts.StructureNode) FilterSearch(com.bakdata.conquery.apiv1.FilterSearch) DatasetRegistry(com.bakdata.conquery.models.worker.DatasetRegistry) Collections(java.util.Collections) InputStream(java.io.InputStream) RemoveTable(com.bakdata.conquery.models.messages.namespaces.specific.RemoveTable) Namespace(com.bakdata.conquery.models.worker.Namespace)

Aggregations

Connector (com.bakdata.conquery.models.datasets.concepts.Connector)10 ConqueryTestSpec (com.bakdata.conquery.integration.json.ConqueryTestSpec)4 Column (com.bakdata.conquery.models.datasets.Column)4 Dataset (com.bakdata.conquery.models.datasets.Dataset)4 DatasetId (com.bakdata.conquery.models.identifiable.ids.specific.DatasetId)4 FilterTemplate (com.bakdata.conquery.apiv1.FilterTemplate)3 SecondaryIdDescription (com.bakdata.conquery.models.datasets.SecondaryIdDescription)3 Table (com.bakdata.conquery.models.datasets.Table)3 Concept (com.bakdata.conquery.models.datasets.concepts.Concept)3 ConceptId (com.bakdata.conquery.models.identifiable.ids.specific.ConceptId)3 IOException (java.io.IOException)3 Getter (lombok.Getter)3 ConceptQuery (com.bakdata.conquery.apiv1.query.ConceptQuery)2 CQTable (com.bakdata.conquery.apiv1.query.concept.filter.CQTable)2 CQUnfilteredTable (com.bakdata.conquery.apiv1.query.concept.filter.CQUnfilteredTable)2 FilterValue (com.bakdata.conquery.apiv1.query.concept.filter.FilterValue)2 CQConcept (com.bakdata.conquery.apiv1.query.concept.specific.CQConcept)2 Jackson (com.bakdata.conquery.io.jackson.Jackson)2 MetaStorage (com.bakdata.conquery.io.storage.MetaStorage)2 Range (com.bakdata.conquery.models.common.Range)2