Examples with Dataset com.bakdata.conquery.models.datasets.Dataset used on opensource projects

Example 31 with Dataset

use of com.bakdata.conquery.models.datasets.Dataset in project conquery by bakdata.

the class SerializationTests method bucketCompoundDateRange.

@Test
public void bucketCompoundDateRange() throws JSONException, IOException {
    Dataset dataset = new Dataset();
    dataset.setName("datasetName");
    Table table = new Table();
    Column startCol = new Column();
    startCol.setName("startCol");
    startCol.setType(MajorTypeId.DATE);
    startCol.setTable(table);
    Column endCol = new Column();
    endCol.setLabel("endLabel");
    endCol.setName("endCol");
    endCol.setType(MajorTypeId.DATE);
    endCol.setTable(table);
    Column compoundCol = new Column();
    compoundCol.setName("compoundCol");
    compoundCol.setType(MajorTypeId.DATE_RANGE);
    compoundCol.setTable(table);
    table.setColumns(new Column[] { startCol, endCol, compoundCol });
    table.setDataset(dataset);
    table.setName("tableName");
    Import imp = new Import(table);
    imp.setName("importTest");
    DateRangeTypeCompound compoundStore = new DateRangeTypeCompound(startCol.getName(), endCol.getName(), new BitSetStore(BitSet.valueOf(new byte[] { 0b1000 }), new BitSet(), 4));
    // 0b1000 is a binary representation of 8 so that the 4th is set to make sure that BitSet length is 4.
    ColumnStore startStore = new IntegerDateStore(new ShortArrayStore(new short[] { 1, 2, 3, 4 }, Short.MIN_VALUE));
    ColumnStore endStore = new IntegerDateStore(new ShortArrayStore(new short[] { 5, 6, 7, 8 }, Short.MIN_VALUE));
    Bucket bucket = new Bucket(0, 1, 4, new ColumnStore[] { startStore, endStore, compoundStore }, Collections.emptySet(), new int[0], new int[0], imp);
    compoundStore.setParent(bucket);
    CentralRegistry registry = new CentralRegistry();
    registry.register(dataset);
    registry.register(startCol);
    registry.register(endCol);
    registry.register(compoundCol);
    registry.register(table);
    registry.register(imp);
    registry.register(bucket);
    final Validator validator = Validators.newValidator();
    SerializationTestUtil.forType(Bucket.class).registry(registry).injectables(new Injectable() {

        @Override
        public MutableInjectableValues inject(MutableInjectableValues values) {
            return values.add(Validator.class, validator);
        }
    }).test(bucket);
}

Example 32 with Dataset

use of com.bakdata.conquery.models.datasets.Dataset in project conquery by bakdata.

the class SerializationTests method managedQuery.

@Test
public void managedQuery() throws JSONException, IOException {
    final CentralRegistry registry = new CentralRegistry();
    final Dataset dataset = new Dataset("test-dataset");
    final User user = new User("test-user", "test-user", STORAGE);
    registry.register(dataset);
    registry.register(user);
    ManagedQuery execution = new ManagedQuery(null, user, dataset);
    execution.setTags(new String[] { "test-tag" });
    SerializationTestUtil.forType(ManagedExecution.class).registry(registry).test(execution);
}

Example 33 with Dataset

use of com.bakdata.conquery.models.datasets.Dataset in project conquery by bakdata.

the class RoleHandlingOnGroupTest method execute.

@Override
public void execute(StandaloneSupport conquery) throws Exception {
    Dataset dataset1 = new Dataset();
    dataset1.setLabel("dataset1");
    MetaStorage storage = conquery.getMetaStorage();
    Group group1 = new Group("company", "company", storage);
    Role role = new Role("role1", "role1", storage);
    TestUser user1 = new TestUser(storage);
    try {
        storage.addRole(role);
        storage.addUser(user1);
        storage.addGroup(group1);
        role.addPermission(new DatasetPermission().instancePermission(Ability.READ, new DatasetId("testDataset")));
        // // Add user to group
        group1.addMember(user1);
        assertThat(user1.isPermitted(new DatasetPermission().instancePermission(Ability.READ, new DatasetId("testDataset")))).isFalse();
        // // Add role to group
        group1.addRole(role);
        assertThat(group1.getRoles()).containsExactlyInAnyOrder(role.getId());
        assertThat(user1.isPermitted(new DatasetPermission().instancePermission(Ability.READ, new DatasetId("testDataset")))).isTrue();
        // // Remove role from group
        group1.removeRole(role);
        assertThat(group1.getRoles()).isEmpty();
        assertThat(user1.isPermitted(new DatasetPermission().instancePermission(Ability.READ, new DatasetId("testDataset")))).isFalse();
    } finally {
        storage.removeGroup(group1.getId());
        storage.removeUser(user1.getId());
        storage.removeRole(role.getId());
    }
}

Example 34 with Dataset

use of com.bakdata.conquery.models.datasets.Dataset in project conquery by bakdata.

the class ConceptUpdateAndDeletionTest method execute.

@Override
public void execute(String name, TestConquery testConquery) throws Exception {
    StandaloneSupport conquery = testConquery.getSupport(name);
    // Read two JSONs with different Trees
    final String testJson = In.resource("/tests/query/UPDATE_CONCEPT_TESTS/SIMPLE_TREECONCEPT_Query.json").withUTF8().readAll();
    final String testJson2 = In.resource("/tests/query/UPDATE_CONCEPT_TESTS/SIMPLE_TREECONCEPT_2_Query.json").withUTF8().readAll();
    final Dataset dataset = conquery.getDataset();
    final Namespace namespace = conquery.getNamespace();
    final ConceptId conceptId = ConceptId.Parser.INSTANCE.parse(dataset.getName(), "test_tree");
    final Concept<?> concept;
    final QueryTest test = (QueryTest) JsonIntegrationTest.readJson(dataset, testJson);
    final QueryTest test2 = (QueryTest) JsonIntegrationTest.readJson(dataset, testJson2);
    // Manually import data, so we can do our own work.
    {
        ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
        importSecondaryIds(conquery, test.getContent().getSecondaryIds());
        conquery.waitUntilWorkDone();
        LoadingUtil.importTables(conquery, test.getContent().getTables());
        conquery.waitUntilWorkDone();
        LoadingUtil.importConcepts(conquery, test.getRawConcepts());
        conquery.waitUntilWorkDone();
        assertThat(namespace.getStorage().getConcept(conceptId)).isNotNull();
        LoadingUtil.importTableContents(conquery, test.getContent().getTables());
        conquery.waitUntilWorkDone();
    }
    final Query query = IntegrationUtils.parseQuery(conquery, test.getRawQuery());
    // State before update.
    {
        log.info("Checking state before update");
        // Must contain the concept.
        assertThat(namespace.getStorage().getAllConcepts()).filteredOn(con -> con.getId().equals(conceptId)).isNotEmpty();
        assertThat(namespace.getStorage().getCentralRegistry().getOptional(conceptId)).isNotEmpty();
        for (ShardNode node : conquery.getShardNodes()) {
            for (Worker value : node.getWorkers().getWorkers().values()) {
                if (!value.getInfo().getDataset().equals(dataset.getId())) {
                    continue;
                }
                final ModificationShieldedWorkerStorage workerStorage = value.getStorage();
                assertThat(workerStorage.getCentralRegistry().getOptional(conceptId)).isNotEmpty();
                assertThat(workerStorage.getAllCBlocks()).describedAs("CBlocks for Worker %s", value.getInfo().getId()).filteredOn(cBlock -> cBlock.getConnector().getConcept().getId().equals(conceptId)).isNotEmpty();
            }
        }
        log.info("Executing query before update");
        IntegrationUtils.assertQueryResult(conquery, query, 1L, ExecutionState.DONE, conquery.getTestUser(), 201);
        conquery.waitUntilWorkDone();
        log.info("Query before update executed");
    }
    // Load a different concept with the same id (it has different children "C1" that are more than "A1")
    // To perform the update, the old concept will be deleted first and the new concept will be added. That means the deletion of concept is also covered here
    {
        log.info("Executing  update");
        LoadingUtil.updateConcepts(conquery, test2.getRawConcepts(), Response.Status.Family.SUCCESSFUL);
        conquery.waitUntilWorkDone();
        log.info("Update executed");
    }
    // Check state after update.
    {
        log.info("Checking state after update");
        // Must contain the concept now.
        assertThat(namespace.getStorage().getAllConcepts()).filteredOn(con -> con.getId().equals(conceptId)).isNotEmpty();
        assertThat(namespace.getStorage().getCentralRegistry().getOptional(conceptId)).isNotEmpty();
        for (ShardNode node : conquery.getShardNodes()) {
            for (Worker value : node.getWorkers().getWorkers().values()) {
                if (!value.getInfo().getDataset().equals(dataset.getId())) {
                    continue;
                }
                final ModificationShieldedWorkerStorage workerStorage = value.getStorage();
                assertThat(workerStorage.getCentralRegistry().getOptional(conceptId)).isNotEmpty();
                assertThat(workerStorage.getAllCBlocks()).describedAs("CBlocks for Worker %s", value.getInfo().getId()).filteredOn(cBlock -> cBlock.getConnector().getConcept().getId().equals(conceptId)).isNotEmpty();
            }
        }
        log.info("Executing query after update");
        // Assert that it now contains 2 instead of 1.
        IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201);
        conquery.waitUntilWorkDone();
        log.info("Query after update executed");
    }
    // new Conquery generated after restarting
    // StandaloneSupport conquery;
    // Restart conquery and assert again, that the data is correct.
    {
        testConquery.shutdown();
        // restart
        testConquery.beforeAll();
        conquery = testConquery.openDataset(dataset.getId());
        log.info("Checking state after re-start");
        {
            // Must contain the concept.
            assertThat(conquery.getNamespace().getStorage().getAllConcepts()).filteredOn(con -> con.getId().equals(conceptId)).isNotEmpty();
            assertThat(conquery.getNamespace().getStorage().getCentralRegistry().getOptional(conceptId)).isNotEmpty();
            for (ShardNode node : conquery.getShardNodes()) {
                for (Worker value : node.getWorkers().getWorkers().values()) {
                    if (!value.getInfo().getDataset().equals(dataset.getId())) {
                        continue;
                    }
                    final ModificationShieldedWorkerStorage workerStorage = value.getStorage();
                    assertThat(workerStorage.getCentralRegistry().getOptional(conceptId)).isNotEmpty();
                    assertThat(workerStorage.getAllCBlocks()).describedAs("CBlocks for Worker %s", value.getInfo().getId()).filteredOn(cBlock -> cBlock.getConnector().getConcept().getId().equals(conceptId)).isNotEmpty();
                }
            }
            log.info("Executing query after restart.");
            // Re-assert state.
            IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201);
            conquery.waitUntilWorkDone();
        }
    }
    // Delete the Concept.
    {
        log.info("Issuing deletion of import {}", conceptId);
        concept = Objects.requireNonNull(conquery.getNamespace().getStorage().getConcept(conceptId));
        conquery.getDatasetsProcessor().deleteConcept(concept);
        conquery.waitUntilWorkDone();
    }
    // Check state after deletion.
    {
        log.info("Checking state after deletion");
        // We've deleted the concept so it and it's associated cblock should be gone.
        assertThat(conquery.getNamespace().getStorage().getAllConcepts()).filteredOn(con -> con.getId().equals(conceptId)).isEmpty();
        assertThat(conquery.getNamespace().getStorage().getCentralRegistry().getOptional(conceptId)).isEmpty();
        assertThat(conquery.getShardNodes().stream().flatMap(node -> node.getWorkers().getWorkers().values().stream()).filter(worker -> worker.getInfo().getDataset().equals(dataset.getId())).map(Worker::getStorage)).noneMatch(workerStorage -> workerStorage.getConcept(conceptId) != null).noneMatch(workerStorage -> workerStorage.getAllCBlocks().stream().anyMatch(cBlock -> cBlock.getConnector().getConcept().getId().equals(conceptId)));
        log.info("Executing query after deletion (EXPECTING AN EXCEPTION IN THE LOGS!)");
        // Issue a query and assert that it is failing.
        IntegrationUtils.assertQueryResult(conquery, query, 0L, ExecutionState.FAILED, conquery.getTestUser(), 400);
    }
    // Restart conquery and assert again, that the state after deletion was maintained.
    {
        {
            testConquery.shutdown();
            // restart
            testConquery.beforeAll();
            conquery = testConquery.openDataset(dataset.getId());
        }
        // Check state after restart.
        {
            log.info("Checking state after restart");
            // We've deleted the concept so it and it's associated cblock should be gone.
            assertThat(conquery.getNamespace().getStorage().getAllConcepts()).filteredOn(con -> con.getId().equals(conceptId)).isEmpty();
            assertThat(conquery.getNamespace().getStorage().getCentralRegistry().getOptional(conceptId)).isEmpty();
            assertThat(conquery.getShardNodes().stream().flatMap(node -> node.getWorkers().getWorkers().values().stream()).filter(worker -> worker.getInfo().getDataset().equals(dataset.getId())).map(Worker::getStorage)).noneMatch(workerStorage -> workerStorage.getConcept(conceptId) != null).noneMatch(workerStorage -> workerStorage.getAllCBlocks().stream().anyMatch(cBlock -> cBlock.getConnector().getConcept().getId().equals(conceptId)));
            log.info("Executing query after restart (EXPECTING AN EXCEPTION IN THE LOGS!)");
            // Issue a query and assert that it is failing.
            IntegrationUtils.assertQueryResult(conquery, query, 0L, ExecutionState.FAILED, conquery.getTestUser(), 400);
        }
    }
}

Example 35 with Dataset

use of com.bakdata.conquery.models.datasets.Dataset in project conquery by bakdata.

the class ImportDeletionTest method execute.

@Override
public void execute(String name, TestConquery testConquery) throws Exception {
    final StandaloneSupport conquery = testConquery.getSupport(name);
    MetaStorage storage = conquery.getMetaStorage();
    final String testJson = In.resource("/tests/query/DELETE_IMPORT_TESTS/SIMPLE_TREECONCEPT_Query.test.json").withUTF8().readAll();
    final Dataset dataset = conquery.getDataset();
    final Namespace namespace = conquery.getNamespace();
    final ImportId importId = ImportId.Parser.INSTANCE.parse(dataset.getName(), "test_table2", "test_table2");
    final QueryTest test = (QueryTest) JsonIntegrationTest.readJson(dataset, testJson);
    // Manually import data, so we can do our own work.
    {
        ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
        importSecondaryIds(conquery, test.getContent().getSecondaryIds());
        conquery.waitUntilWorkDone();
        LoadingUtil.importTables(conquery, test.getContent().getTables());
        conquery.waitUntilWorkDone();
        LoadingUtil.importConcepts(conquery, test.getRawConcepts());
        conquery.waitUntilWorkDone();
        LoadingUtil.importTableContents(conquery, test.getContent().getTables());
        conquery.waitUntilWorkDone();
    }
    final Query query = IntegrationUtils.parseQuery(conquery, test.getRawQuery());
    final int nImports = namespace.getStorage().getAllImports().size();
    // State before deletion.
    {
        log.info("Checking state before deletion");
        // Must contain the import.
        assertThat(namespace.getStorage().getAllImports()).filteredOn(imp -> imp.getId().equals(importId)).isNotEmpty();
        assertThat(namespace.getStorage().getCentralRegistry().getOptional(importId)).isNotEmpty();
        for (ShardNode node : conquery.getShardNodes()) {
            for (Worker worker : node.getWorkers().getWorkers().values()) {
                if (!worker.getInfo().getDataset().equals(dataset.getId())) {
                    continue;
                }
                final ModificationShieldedWorkerStorage workerStorage = worker.getStorage();
                assertThat(workerStorage.getAllCBlocks()).describedAs("CBlocks for Worker %s", worker.getInfo().getId()).filteredOn(block -> block.getBucket().getId().getDataset().equals(dataset.getId())).isNotEmpty();
                assertThat(workerStorage.getAllBuckets()).filteredOn(bucket -> bucket.getId().getDataset().equals(dataset.getId())).describedAs("Buckets for Worker %s", worker.getInfo().getId()).isNotEmpty();
                // Must contain the import.
                assertThat(workerStorage.getImport(importId)).isNotNull();
            }
        }
        log.info("Executing query before deletion");
        IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201);
    }
    // Delete the import.
    {
        log.info("Issuing deletion of import {}", importId);
        final URI deleteImportUri = HierarchyHelper.hierarchicalPath(conquery.defaultAdminURIBuilder(), AdminTablesResource.class, "deleteImport").buildFromMap(Map.of(ResourceConstants.DATASET, conquery.getDataset().getId(), ResourceConstants.TABLE, importId.getTable(), ResourceConstants.IMPORT_ID, importId));
        final Response delete = conquery.getClient().target(deleteImportUri).request(MediaType.APPLICATION_JSON).delete();
        assertThat(delete.getStatusInfo().getFamily()).isEqualTo(Response.Status.Family.SUCCESSFUL);
        conquery.waitUntilWorkDone();
    }
    // State after deletion.
    {
        log.info("Checking state after deletion");
        // We have deleted an import now there should be one less!
        assertThat(namespace.getStorage().getAllImports().size()).isEqualTo(nImports - 1);
        // The deleted import should not be found.
        assertThat(namespace.getStorage().getAllImports()).filteredOn(imp -> imp.getId().equals(importId)).isEmpty();
        for (ShardNode node : conquery.getShardNodes()) {
            for (Worker worker : node.getWorkers().getWorkers().values()) {
                if (!worker.getInfo().getDataset().equals(dataset.getId())) {
                    continue;
                }
                final ModificationShieldedWorkerStorage workerStorage = worker.getStorage();
                // No bucket should be found referencing the import.
                assertThat(workerStorage.getAllBuckets()).describedAs("Buckets for Worker %s", worker.getInfo().getId()).filteredOn(bucket -> bucket.getImp().getId().equals(importId)).isEmpty();
                // No CBlock associated with import may exist
                assertThat(workerStorage.getAllCBlocks()).describedAs("CBlocks for Worker %s", worker.getInfo().getId()).filteredOn(cBlock -> cBlock.getBucket().getId().getImp().equals(importId)).isEmpty();
                // Import should not exists anymore
                assertThat(workerStorage.getImport(importId)).describedAs("Import for Worker %s", worker.getInfo().getId()).isNull();
            }
        }
        log.info("Executing query after deletion");
        // Issue a query and assert that it has less content.
        IntegrationUtils.assertQueryResult(conquery, query, 1L, ExecutionState.DONE, conquery.getTestUser(), 201);
    }
    conquery.waitUntilWorkDone();
    // Load more data under the same name into the same table, with only the deleted import/table
    {
        // only import the deleted import/table
        final RequiredTable import2Table = test.getContent().getTables().stream().filter(table -> table.getName().equalsIgnoreCase(importId.getTable().getTable())).findFirst().orElseThrow();
        final ResourceFile csv = import2Table.getCsv();
        final String path = csv.getPath();
        // copy csv to tmp folder
        // Content 2.2 contains an extra entry of a value that hasn't been seen before.
        FileUtils.copyInputStreamToFile(In.resource(path.substring(0, path.lastIndexOf('/')) + "/" + "content2.2.csv").asStream(), new File(conquery.getTmpDir(), csv.getName()));
        File descriptionFile = new File(conquery.getTmpDir(), import2Table.getName() + ConqueryConstants.EXTENSION_DESCRIPTION);
        File preprocessedFile = new File(conquery.getTmpDir(), import2Table.getName() + ConqueryConstants.EXTENSION_PREPROCESSED);
        // create import descriptor
        TableImportDescriptor desc = new TableImportDescriptor();
        desc.setName(import2Table.getName());
        desc.setTable(import2Table.getName());
        TableInputDescriptor input = new TableInputDescriptor();
        {
            input.setPrimary(import2Table.getPrimaryColumn().createOutput());
            input.setSourceFile(import2Table.getCsv().getName());
            input.setOutput(new OutputDescription[import2Table.getColumns().length]);
            for (int i = 0; i < import2Table.getColumns().length; i++) {
                input.getOutput()[i] = import2Table.getColumns()[i].createOutput();
            }
        }
        desc.setInputs(new TableInputDescriptor[] { input });
        Jackson.MAPPER.writeValue(descriptionFile, desc);
        // preprocess
        conquery.preprocessTmp(conquery.getTmpDir(), List.of(descriptionFile));
        // import preprocessedFiles
        conquery.getDatasetsProcessor().addImport(conquery.getNamespace(), new GZIPInputStream(new FileInputStream(preprocessedFile)));
        conquery.waitUntilWorkDone();
    }
    // State after reimport.
    {
        log.info("Checking state after re-import");
        assertThat(namespace.getStorage().getAllImports().size()).isEqualTo(nImports);
        for (ShardNode node : conquery.getShardNodes()) {
            for (Worker worker : node.getWorkers().getWorkers().values()) {
                if (!worker.getInfo().getDataset().equals(dataset.getId())) {
                    continue;
                }
                final ModificationShieldedWorkerStorage workerStorage = worker.getStorage();
                assertThat(workerStorage.getAllBuckets()).describedAs("Buckets for Worker %s", worker.getInfo().getId()).filteredOn(bucket -> bucket.getImp().getId().equals(importId)).filteredOn(bucket -> bucket.getId().getDataset().equals(dataset.getId())).isNotEmpty();
            }
        }
        log.info("Executing query after re-import");
        // Issue a query and assert that it has the same content as the first time around.
        IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201);
    }
    // Finally, restart conquery and assert again, that the data is correct.
    {
        testConquery.shutdown();
        // restart
        testConquery.beforeAll();
        StandaloneSupport conquery2 = testConquery.openDataset(dataset.getId());
        log.info("Checking state after re-start");
        {
            assertThat(namespace.getStorage().getAllImports().size()).isEqualTo(2);
            for (ShardNode node : conquery2.getShardNodes()) {
                for (Worker worker : node.getWorkers().getWorkers().values()) {
                    if (!worker.getInfo().getDataset().equals(dataset.getId()))
                        continue;
                    final ModificationShieldedWorkerStorage workerStorage = worker.getStorage();
                    assertThat(workerStorage.getAllBuckets()).describedAs("Buckets for Worker %s", worker.getInfo().getId()).filteredOn(bucket -> bucket.getId().getDataset().equals(dataset.getId())).filteredOn(bucket -> bucket.getImp().getId().equals(importId)).isNotEmpty();
                }
            }
            log.info("Executing query after re-import");
            // Issue a query and assert that it has the same content as the first time around.
            IntegrationUtils.assertQueryResult(conquery2, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201);
        }
    }
}