Search in sources :

Example 1 with ConqueryTestSpec

use of com.bakdata.conquery.integration.json.ConqueryTestSpec in project conquery by bakdata.

the class FilterResolutionExactTest method execute.

@Override
public void execute(StandaloneSupport conquery) throws Exception {
    // read test sepcification
    String testJson = In.resource("/tests/query/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY.test.json").withUTF8().readAll();
    DatasetId dataset = conquery.getDataset().getId();
    ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson);
    ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
    CSVConfig csvConf = conquery.getConfig().getCsv();
    test.importRequiredData(conquery);
    FilterSearch.updateSearch(conquery.getNamespace().getNamespaces(), Collections.singleton(conquery.getNamespace().getDataset()), conquery.getDatasetsProcessor().getJobManager(), csvConf);
    conquery.waitUntilWorkDone();
    Concept<?> concept = conquery.getNamespace().getStorage().getAllConcepts().iterator().next();
    Connector connector = concept.getConnectors().iterator().next();
    AbstractSelectFilter<?> filter = (AbstractSelectFilter<?>) connector.getFilters().iterator().next();
    // Copy search csv from resources to tmp folder.
    final Path tmpCSv = Files.createTempFile("conquery_search", "csv");
    Files.write(tmpCSv, String.join(csvConf.getLineSeparator(), lines).getBytes(), StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
    filter.setSearchType(FilterSearch.FilterSearchType.EXACT);
    filter.setTemplate(new FilterTemplate(tmpCSv.toString(), Arrays.asList("HEADER"), "HEADER", "", ""));
    FilterSearch.createSourceSearch(filter, csvConf);
    assertThat(filter.getSourceSearch()).isNotNull();
    ConceptsProcessor processor = new ConceptsProcessor(conquery.getNamespace().getNamespaces());
    // from csv
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("a", "aaa", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("a", "aaa");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
    // from column values
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("f", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("f");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
}
Also used : Path(java.nio.file.Path) Connector(com.bakdata.conquery.models.datasets.concepts.Connector) AbstractSelectFilter(com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter) ConqueryTestSpec(com.bakdata.conquery.integration.json.ConqueryTestSpec) ConceptsProcessor(com.bakdata.conquery.resources.api.ConceptsProcessor) CSVConfig(com.bakdata.conquery.models.config.CSVConfig) DatasetId(com.bakdata.conquery.models.identifiable.ids.specific.DatasetId) FilterTemplate(com.bakdata.conquery.apiv1.FilterTemplate) ResolvedConceptsResult(com.bakdata.conquery.resources.api.ConceptsProcessor.ResolvedConceptsResult)

Example 2 with ConqueryTestSpec

use of com.bakdata.conquery.integration.json.ConqueryTestSpec in project conquery by bakdata.

the class FilterResolutionPrefixTest method execute.

@Override
public void execute(StandaloneSupport conquery) throws Exception {
    // read test specification
    String testJson = In.resource("/tests/query/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY.test.json").withUTF8().readAll();
    DatasetId dataset = conquery.getDataset().getId();
    ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson);
    ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
    test.importRequiredData(conquery);
    CSVConfig csvConf = conquery.getConfig().getCsv();
    FilterSearch.updateSearch(conquery.getNamespace().getNamespaces(), Collections.singleton(conquery.getNamespace().getDataset()), conquery.getDatasetsProcessor().getJobManager(), csvConf);
    conquery.waitUntilWorkDone();
    Concept<?> concept = conquery.getNamespace().getStorage().getAllConcepts().iterator().next();
    Connector connector = concept.getConnectors().iterator().next();
    AbstractSelectFilter<?> filter = (AbstractSelectFilter<?>) connector.getFilters().iterator().next();
    // Copy search csv from resources to tmp folder.
    final Path tmpCSv = Files.createTempFile("conquery_search", "csv");
    Files.write(tmpCSv, String.join(csvConf.getLineSeparator(), lines).getBytes(), StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
    filter.setSearchType(FilterSearch.FilterSearchType.PREFIX);
    filter.setTemplate(new FilterTemplate(tmpCSv.toString(), Arrays.asList("HEADER"), "HEADER", "", ""));
    FilterSearch.createSourceSearch(filter, csvConf);
    assertThat(filter.getSourceSearch()).isNotNull();
    ConceptsProcessor processor = new ConceptsProcessor(conquery.getNamespace().getNamespaces());
    // from csv
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("a", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("a", "aaa", "aab");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
    // from column values
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("f", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("f");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
}
Also used : Path(java.nio.file.Path) Connector(com.bakdata.conquery.models.datasets.concepts.Connector) AbstractSelectFilter(com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter) ConqueryTestSpec(com.bakdata.conquery.integration.json.ConqueryTestSpec) ConceptsProcessor(com.bakdata.conquery.resources.api.ConceptsProcessor) CSVConfig(com.bakdata.conquery.models.config.CSVConfig) DatasetId(com.bakdata.conquery.models.identifiable.ids.specific.DatasetId) FilterTemplate(com.bakdata.conquery.apiv1.FilterTemplate) ResolvedConceptsResult(com.bakdata.conquery.resources.api.ConceptsProcessor.ResolvedConceptsResult)

Example 3 with ConqueryTestSpec

use of com.bakdata.conquery.integration.json.ConqueryTestSpec in project conquery by bakdata.

the class MetadataCollectionTest method execute.

@Override
public void execute(StandaloneSupport conquery) throws Exception {
    // read test sepcification
    String testJson = In.resource("/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json").withUTF8().readAll();
    DatasetId dataset = conquery.getDataset().getId();
    ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson);
    ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
    test.importRequiredData(conquery);
    // ensure the metadata is collected
    conquery.getNamespace().sendToAll(new UpdateMatchingStatsMessage());
    conquery.waitUntilWorkDone();
    TreeConcept concept = (TreeConcept) conquery.getNamespace().getStorage().getAllConcepts().iterator().next();
    // check the number of matched events
    assertThat(concept.getMatchingStats().countEvents()).isEqualTo(4);
    assertThat(concept.getChildren()).allSatisfy(c -> {
        assertThat(c.getMatchingStats().countEvents()).isEqualTo(2);
    });
    // check the date ranges
    assertThat(concept.getMatchingStats().spanEvents()).isEqualTo(CDateRange.of(LocalDate.parse("2010-07-15"), LocalDate.parse("2013-11-10")));
    assertThat(concept.getChildren().get(0).getMatchingStats().spanEvents()).isEqualTo(CDateRange.of(LocalDate.parse("2012-01-01"), LocalDate.parse("2013-11-10")));
    assertThat(concept.getChildren().get(1).getMatchingStats().spanEvents()).isEqualTo(CDateRange.of(LocalDate.parse("2010-07-15"), LocalDate.parse("2012-11-11")));
}
Also used : UpdateMatchingStatsMessage(com.bakdata.conquery.models.messages.namespaces.specific.UpdateMatchingStatsMessage) TreeConcept(com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept) ConqueryTestSpec(com.bakdata.conquery.integration.json.ConqueryTestSpec) DatasetId(com.bakdata.conquery.models.identifiable.ids.specific.DatasetId)

Example 4 with ConqueryTestSpec

use of com.bakdata.conquery.integration.json.ConqueryTestSpec in project conquery by bakdata.

the class ConceptResolutionTest method execute.

@Override
public void execute(StandaloneSupport conquery) throws Exception {
    // read test sepcification
    String testJson = In.resource("/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json").withUTF8().readAll();
    DatasetId dataset = conquery.getDataset().getId();
    ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson);
    ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
    test.importRequiredData(conquery);
    FilterSearch.updateSearch(conquery.getNamespace().getNamespaces(), Collections.singleton(conquery.getNamespace().getDataset()), conquery.getDatasetsProcessor().getJobManager(), conquery.getConfig().getCsv());
    conquery.waitUntilWorkDone();
    ConceptsProcessor processor = new ConceptsProcessor(conquery.getNamespace().getNamespaces());
    TreeConcept concept = (TreeConcept) conquery.getNamespace().getStorage().getAllConcepts().iterator().next();
    ResolvedConceptsResult resolved = processor.resolveConceptElements(concept, List.of("A1", "unknown"));
    // check the resolved values
    assertThat(resolved).isNotNull();
    assertThat(resolved.getResolvedConcepts().stream().map(IId::toString)).containsExactlyInAnyOrder("ConceptResolutionTest.test_tree.test_child1");
    assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
}
Also used : IId(com.bakdata.conquery.models.identifiable.ids.IId) TreeConcept(com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept) ConqueryTestSpec(com.bakdata.conquery.integration.json.ConqueryTestSpec) ConceptsProcessor(com.bakdata.conquery.resources.api.ConceptsProcessor) DatasetId(com.bakdata.conquery.models.identifiable.ids.specific.DatasetId) ResolvedConceptsResult(com.bakdata.conquery.resources.api.ConceptsProcessor.ResolvedConceptsResult)

Example 5 with ConqueryTestSpec

use of com.bakdata.conquery.integration.json.ConqueryTestSpec in project conquery by bakdata.

the class FilterResolutionContainsTest method execute.

@Override
public void execute(StandaloneSupport conquery) throws Exception {
    // read test sepcification
    String testJson = In.resource("/tests/query/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY.test.json").withUTF8().readAll();
    DatasetId dataset = conquery.getDataset().getId();
    ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson);
    ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
    test.importRequiredData(conquery);
    CSVConfig csvConf = conquery.getConfig().getCsv();
    FilterSearch.updateSearch(conquery.getNamespace().getNamespaces(), Collections.singleton(conquery.getNamespace().getDataset()), conquery.getDatasetsProcessor().getJobManager(), csvConf);
    conquery.waitUntilWorkDone();
    Concept<?> concept = conquery.getNamespace().getStorage().getAllConcepts().iterator().next();
    Connector connector = concept.getConnectors().iterator().next();
    AbstractSelectFilter<?> filter = (AbstractSelectFilter<?>) connector.getFilters().iterator().next();
    // Copy search csv from resources to tmp folder.
    final Path tmpCSv = Files.createTempFile("conquery_search", ".csv");
    Out.file(tmpCSv.toFile()).withUTF8().writeLines(lines);
    Files.write(tmpCSv, String.join(csvConf.getLineSeparator(), lines).getBytes(), StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
    filter.setSearchType(FilterSearch.FilterSearchType.CONTAINS);
    filter.setTemplate(new FilterTemplate(tmpCSv.toString(), Arrays.asList("HEADER"), "HEADER", "", ""));
    FilterSearch.createSourceSearch(filter, csvConf);
    assertThat(filter.getSourceSearch()).isNotNull();
    ConceptsProcessor processor = new ConceptsProcessor(conquery.getNamespace().getNamespaces());
    // from csv
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("a", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("a", "aaa", "aab", "baaa");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
    // from column values
    {
        ResolvedConceptsResult resolved = processor.resolveFilterValues(filter, List.of("f", "unknown"));
        // check the resolved values
        assertThat(resolved.getResolvedFilter().getValue().stream().map(FEValue::getValue)).containsExactlyInAnyOrder("f");
        assertThat(resolved.getUnknownCodes()).containsExactlyInAnyOrder("unknown");
    }
}
Also used : Path(java.nio.file.Path) Connector(com.bakdata.conquery.models.datasets.concepts.Connector) AbstractSelectFilter(com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter) ConqueryTestSpec(com.bakdata.conquery.integration.json.ConqueryTestSpec) ConceptsProcessor(com.bakdata.conquery.resources.api.ConceptsProcessor) CSVConfig(com.bakdata.conquery.models.config.CSVConfig) DatasetId(com.bakdata.conquery.models.identifiable.ids.specific.DatasetId) FilterTemplate(com.bakdata.conquery.apiv1.FilterTemplate) ResolvedConceptsResult(com.bakdata.conquery.resources.api.ConceptsProcessor.ResolvedConceptsResult)

Aggregations

ConqueryTestSpec (com.bakdata.conquery.integration.json.ConqueryTestSpec)6 DatasetId (com.bakdata.conquery.models.identifiable.ids.specific.DatasetId)6 ConceptsProcessor (com.bakdata.conquery.resources.api.ConceptsProcessor)4 ResolvedConceptsResult (com.bakdata.conquery.resources.api.ConceptsProcessor.ResolvedConceptsResult)4 FilterTemplate (com.bakdata.conquery.apiv1.FilterTemplate)3 CSVConfig (com.bakdata.conquery.models.config.CSVConfig)3 Connector (com.bakdata.conquery.models.datasets.concepts.Connector)3 AbstractSelectFilter (com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter)3 Path (java.nio.file.Path)3 TreeConcept (com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept)2 ManagerNode (com.bakdata.conquery.commands.ManagerNode)1 MetaStorage (com.bakdata.conquery.io.storage.MetaStorage)1 NamespaceStorage (com.bakdata.conquery.io.storage.NamespaceStorage)1 Group (com.bakdata.conquery.models.auth.entities.Group)1 Role (com.bakdata.conquery.models.auth.entities.Role)1 User (com.bakdata.conquery.models.auth.entities.User)1 Dataset (com.bakdata.conquery.models.datasets.Dataset)1 IId (com.bakdata.conquery.models.identifiable.ids.IId)1 EntityIdMap (com.bakdata.conquery.models.identifiable.mapping.EntityIdMap)1 UpdateMatchingStatsMessage (com.bakdata.conquery.models.messages.namespaces.specific.UpdateMatchingStatsMessage)1