Examples with Worker com.bakdata.conquery.models.worker.Worker used on opensource projects

Example 1 with Worker

use of com.bakdata.conquery.models.worker.Worker in project conquery by bakdata.

the class ImportUpdateTest method execute.

@Override
public void execute(String name, TestConquery testConquery) throws Exception {
    final StandaloneSupport conquery = testConquery.getSupport(name);
    MetaStorage storage = conquery.getMetaStorage();
    String testJson = In.resource("/tests/query/UPDATE_IMPORT_TESTS/SIMPLE_TREECONCEPT_Query.json").withUTF8().readAll();
    final Dataset dataset = conquery.getDataset();
    final Namespace namespace = conquery.getNamespace();
    final ImportId importId1 = ImportId.Parser.INSTANCE.parse(dataset.getName(), "table1", "table1");
    final ImportId importId2 = ImportId.Parser.INSTANCE.parse(dataset.getName(), "table2", "table2");
    QueryTest test = (QueryTest) JsonIntegrationTest.readJson(dataset, testJson);
    final List<RequiredTable> tables = test.getContent().getTables();
    assertThat(tables.size()).isEqualTo(2);
    List<File> cqpps;
    // Manually import data, so we can do our own work.
    {
        ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
        importSecondaryIds(conquery, test.getContent().getSecondaryIds());
        conquery.waitUntilWorkDone();
        LoadingUtil.importTables(conquery, tables);
        conquery.waitUntilWorkDone();
        LoadingUtil.importConcepts(conquery, test.getRawConcepts());
        conquery.waitUntilWorkDone();
        cqpps = LoadingUtil.generateCqpp(conquery, tables);
        conquery.waitUntilWorkDone();
        assertThat(cqpps.size()).isEqualTo(tables.size());
        LoadingUtil.importCqppFiles(conquery, List.of(cqpps.get(0)));
        conquery.waitUntilWorkDone();
    }
    final Query query = IntegrationUtils.parseQuery(conquery, test.getRawQuery());
    // State before update.
    {
        log.info("Checking state before update");
        assertThat(namespace.getStorage().getAllImports()).hasSize(1);
        // Must contain the import.
        assertThat(namespace.getStorage().getAllImports()).filteredOn(imp -> imp.getId().equals(importId1)).isNotEmpty();
        assertThat(namespace.getStorage().getCentralRegistry().getOptional(importId1)).isNotEmpty();
        for (ShardNode node : conquery.getShardNodes()) {
            for (Worker worker : node.getWorkers().getWorkers().values()) {
                if (!worker.getInfo().getDataset().equals(dataset.getId())) {
                    continue;
                }
                final ModificationShieldedWorkerStorage workerStorage = worker.getStorage();
                assertThat(workerStorage.getAllCBlocks()).describedAs("CBlocks for Worker %s", worker.getInfo().getId()).filteredOn(block -> block.getBucket().getId().getDataset().equals(dataset.getId())).isNotEmpty();
                assertThat(workerStorage.getAllBuckets()).filteredOn(bucket -> bucket.getId().getDataset().equals(dataset.getId())).describedAs("Buckets for Worker %s", worker.getInfo().getId()).isNotEmpty();
                // Must contain the import.
                assertThat(workerStorage.getImport(importId1)).isNotNull();
            }
        }
        assertThat(namespace.getNumberOfEntities()).isEqualTo(4);
        // assert that the query can be executed after the import
        IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201);
    }
    // Try to update an import that does not exist should throw a Not-Found Webapplication Exception
    LoadingUtil.updateCqppFile(conquery, cqpps.get(1), Response.Status.Family.CLIENT_ERROR, "Not Found");
    conquery.waitUntilWorkDone();
    // Load manually new data for import and update the concerned import
    {
        log.info("Manually loading new data for import");
        final RequiredTable importTable = test.getContent().getTables().stream().filter(table -> table.getName().equalsIgnoreCase(importId1.getTable().getTable())).findFirst().orElseThrow();
        final String csvName = importTable.getCsv().getName();
        final String path = importTable.getCsv().getPath();
        // copy new content of the importTable into the csv-File used by the preprocessor to avoid creating multiple files withe same names
        FileUtils.copyInputStreamToFile(In.resource(path.substring(0, path.lastIndexOf('/')) + "/" + csvName.replace(".csv", ".update.csv")).asStream(), new File(conquery.getTmpDir(), csvName));
        File descriptionFile = new File(conquery.getTmpDir(), importTable.getName() + ConqueryConstants.EXTENSION_DESCRIPTION);
        File newPreprocessedFile = new File(conquery.getTmpDir(), importTable.getName() + ConqueryConstants.EXTENSION_PREPROCESSED);
        // create import descriptor
        {
            TableImportDescriptor desc = new TableImportDescriptor();
            desc.setName(importTable.getName());
            desc.setTable(importTable.getName());
            TableInputDescriptor input = new TableInputDescriptor();
            {
                input.setPrimary(importTable.getPrimaryColumn().createOutput());
                input.setSourceFile(csvName);
                input.setOutput(new OutputDescription[importTable.getColumns().length]);
                for (int i = 0; i < importTable.getColumns().length; i++) {
                    input.getOutput()[i] = importTable.getColumns()[i].createOutput();
                }
            }
            desc.setInputs(new TableInputDescriptor[] { input });
            Jackson.MAPPER.writeValue(descriptionFile, desc);
        }
        // preprocess
        conquery.preprocessTmp(conquery.getTmpDir(), List.of(descriptionFile));
        log.info("updating import");
        // correct update of the import
        LoadingUtil.updateCqppFile(conquery, newPreprocessedFile, Response.Status.Family.SUCCESSFUL, "No Content");
        conquery.waitUntilWorkDone();
    }
    // State after update.
    {
        log.info("Checking state after update");
        assertThat(namespace.getStorage().getAllImports()).hasSize(1);
        // Must contain the import.
        assertThat(namespace.getStorage().getAllImports()).filteredOn(imp -> imp.getId().equals(importId1)).isNotEmpty();
        assertThat(namespace.getStorage().getCentralRegistry().getOptional(importId1)).isNotEmpty();
        for (ShardNode node : conquery.getShardNodes()) {
            for (Worker worker : node.getWorkers().getWorkers().values()) {
                if (!worker.getInfo().getDataset().equals(dataset.getId())) {
                    continue;
                }
                final ModificationShieldedWorkerStorage workerStorage = worker.getStorage();
                assertThat(workerStorage.getAllCBlocks()).describedAs("CBlocks for Worker %s", worker.getInfo().getId()).filteredOn(block -> block.getBucket().getId().getDataset().equals(dataset.getId())).isNotEmpty();
                assertThat(workerStorage.getAllBuckets()).filteredOn(bucket -> bucket.getId().getDataset().equals(dataset.getId())).describedAs("Buckets for Worker %s", worker.getInfo().getId()).isNotEmpty();
                // Must contain the import.
                assertThat(workerStorage.getImport(importId1)).isNotNull();
            }
        }
        assertThat(namespace.getNumberOfEntities()).isEqualTo(9);
        // Issue a query and assert that it has more content.
        IntegrationUtils.assertQueryResult(conquery, query, 4L, ExecutionState.DONE, conquery.getTestUser(), 201);
    }
}

Example 2 with Worker

use of com.bakdata.conquery.models.worker.Worker in project conquery by bakdata.

the class DatasetDeletionTest method execute.

@Override
public void execute(String name, TestConquery testConquery) throws Exception {
    final StandaloneSupport conquery = testConquery.getSupport(name);
    final MetaStorage storage = conquery.getMetaStorage();
    final Dataset dataset = conquery.getDataset();
    Namespace namespace = conquery.getNamespace();
    final String testJson = In.resource("/tests/query/DELETE_IMPORT_TESTS/SIMPLE_TREECONCEPT_Query.test.json").withUTF8().readAll();
    final QueryTest test = (QueryTest) JsonIntegrationTest.readJson(dataset, testJson);
    // Manually import data, so we can do our own work.
    {
        ValidatorHelper.failOnError(log, conquery.getValidator().validate(test));
        importSecondaryIds(conquery, test.getContent().getSecondaryIds());
        conquery.waitUntilWorkDone();
        LoadingUtil.importTables(conquery, test.getContent().getTables());
        conquery.waitUntilWorkDone();
        LoadingUtil.importConcepts(conquery, test.getRawConcepts());
        conquery.waitUntilWorkDone();
        LoadingUtil.importTableContents(conquery, test.getContent().getTables());
        conquery.waitUntilWorkDone();
    }
    final Query query = IntegrationUtils.parseQuery(conquery, test.getRawQuery());
    final int nImports = namespace.getStorage().getAllImports().size();
    log.info("Checking state before deletion");
    // Assert state before deletion.
    {
        // Must contain the import.
        assertThat(namespace.getStorage().getCentralRegistry().getOptional(dataset.getId())).isNotEmpty();
        for (ShardNode node : conquery.getShardNodes()) {
            for (Worker value : node.getWorkers().getWorkers().values()) {
                if (!value.getInfo().getDataset().equals(dataset.getId())) {
                    continue;
                }
                final ModificationShieldedWorkerStorage workerStorage = value.getStorage();
                assertThat(workerStorage.getAllCBlocks()).describedAs("CBlocks for Worker %s", value.getInfo().getId()).isNotEmpty();
                assertThat(workerStorage.getAllBuckets()).describedAs("Buckets for Worker %s", value.getInfo().getId()).isNotEmpty();
            }
        }
        log.info("Executing query before deletion");
        IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201);
    }
    // Delete Dataset.
    {
        log.info("Issuing deletion of import {}", dataset);
        // Delete the import.
        // But, we do not allow deletion of tables with associated connectors, so this should throw!
        assertThatThrownBy(() -> conquery.getDatasetsProcessor().deleteDataset(dataset)).isInstanceOf(WebApplicationException.class);
        // TODO use api
        conquery.getNamespace().getStorage().getTables().forEach(tableId -> conquery.getDatasetsProcessor().deleteTable(tableId, true));
        conquery.waitUntilWorkDone();
        // Finally delete dataset
        conquery.getDatasetsProcessor().deleteDataset(dataset);
        conquery.waitUntilWorkDone();
        assertThat(storage.getCentralRegistry().getOptional(dataset.getId())).isEmpty();
    }
    // State after deletion.
    {
        log.info("Checking state after deletion");
        // We have deleted an import now there should be two less!
        assertThat(namespace.getStorage().getAllImports().size()).isEqualTo(0);
        // The deleted import should not be found.
        assertThat(namespace.getStorage().getAllImports()).filteredOn(imp -> imp.getId().getTable().getDataset().equals(dataset.getId())).isEmpty();
        for (ShardNode node : conquery.getShardNodes()) {
            for (Worker value : node.getWorkers().getWorkers().values()) {
                if (!value.getInfo().getDataset().equals(dataset.getId())) {
                    continue;
                }
                final ModificationShieldedWorkerStorage workerStorage = value.getStorage();
                // No bucket should be found referencing the import.
                assertThat(workerStorage.getAllBuckets()).describedAs("Buckets for Worker %s", value.getInfo().getId()).filteredOn(bucket -> bucket.getTable().getDataset().getId().equals(dataset.getId())).isEmpty();
                // No CBlock associated with import may exist
                assertThat(workerStorage.getAllCBlocks()).describedAs("CBlocks for Worker %s", value.getInfo().getId()).filteredOn(cBlock -> cBlock.getBucket().getTable().getDataset().getId().equals(dataset.getId())).isEmpty();
            }
        }
        // It's not exactly possible to issue a query for a non-existant dataset, so we assert that parsing the fails.
        assertThatThrownBy(() -> {
            IntegrationUtils.parseQuery(conquery, test.getRawQuery());
        }).isNotNull();
        IntegrationUtils.assertQueryResult(conquery, query, 0, ExecutionState.FAILED, conquery.getTestUser(), 404);
    }
    // Reload the dataset and assert the state.
    // We have to do some weird trix with StandaloneSupport to open it with another Dataset
    final StandaloneSupport conqueryReimport = testConquery.getSupport(namespace.getDataset().getName());
    {
        // only import the deleted import/table
        LoadingUtil.importTables(conqueryReimport, test.getContent().getTables());
        assertThat(conqueryReimport.getNamespace().getStorage().getTables()).isNotEmpty();
        conqueryReimport.waitUntilWorkDone();
        LoadingUtil.importTableContents(conqueryReimport, test.getContent().getTables());
        conqueryReimport.waitUntilWorkDone();
        LoadingUtil.importConcepts(conqueryReimport, test.getRawConcepts());
        conqueryReimport.waitUntilWorkDone();
        assertThat(conqueryReimport.getDatasetsProcessor().getDatasetRegistry().get(conqueryReimport.getDataset().getId())).describedAs("Dataset after re-import.").isNotNull();
        assertThat(conqueryReimport.getNamespace().getStorage().getAllImports().size()).isEqualTo(nImports);
        for (ShardNode node : conqueryReimport.getShardNodes()) {
            assertThat(node.getWorkers().getWorkers().values()).filteredOn(w -> w.getInfo().getDataset().equals(conqueryReimport.getDataset().getId())).describedAs("Workers for node {}", node.getName()).isNotEmpty();
        }
        log.info("Executing query after re-import");
        final Query query2 = IntegrationUtils.parseQuery(conqueryReimport, test.getRawQuery());
        // Issue a query and assert that it has the same content as the first time around.
        IntegrationUtils.assertQueryResult(conqueryReimport, query2, 2L, ExecutionState.DONE, conqueryReimport.getTestUser(), 201);
    }
    // Finally, restart conquery and assert again, that the data is correct.
    {
        testConquery.shutdown();
        // restart
        testConquery.beforeAll();
        final StandaloneSupport conqueryRestart = testConquery.openDataset(conqueryReimport.getDataset().getId());
        log.info("Checking state after re-start");
        assertThat(conqueryRestart.getNamespace().getStorage().getAllImports().size()).isEqualTo(2);
        for (ShardNode node : conqueryRestart.getShardNodes()) {
            for (Worker value : node.getWorkers().getWorkers().values()) {
                if (!value.getInfo().getDataset().equals(dataset.getId())) {
                    continue;
                }
                final ModificationShieldedWorkerStorage workerStorage = value.getStorage();
                assertThat(workerStorage.getAllBuckets().stream().filter(bucket -> bucket.getTable().getDataset().getId().equals(dataset.getId()))).describedAs("Buckets for Worker %s", value.getInfo().getId()).isNotEmpty();
            }
        }
        log.info("Executing query after restart");
        final Query query3 = IntegrationUtils.parseQuery(conqueryRestart, test.getRawQuery());
        // Issue a query and assert that it has the same content as the first time around.
        IntegrationUtils.assertQueryResult(conqueryRestart, query3, 2L, ExecutionState.DONE, conquery.getTestUser(), 201);
    }
}

Example 3 with Worker

use of com.bakdata.conquery.models.worker.Worker in project conquery by bakdata.

the class ForwardToWorker method react.

@Override
public void react(ShardNodeNetworkContext context) throws Exception {
    Worker worker = Objects.requireNonNull(context.getWorkers().getWorker(workerId));
    ConqueryMDC.setLocation(worker.toString());
    WorkerMessage message = deserializeMessage(messageRaw, context.getWorkers().getBinaryMapper());
    if (message instanceof SlowMessage) {
        ((SlowMessage) message).setProgressReporter(progressReporter);
    }
    message.react(worker);
}

Example 4 with Worker

use of com.bakdata.conquery.models.worker.Worker in project conquery by bakdata.

the class ShardNode method start.

@Override
public void start() throws Exception {
    for (Worker value : workers.getWorkers().values()) {
        value.getJobManager().addSlowJob(new SimpleJob("Update Bucket Manager", value.getBucketManager()::fullUpdate));
    }
    ObjectMapper om = Jackson.copyMapperAndInjectables(Jackson.BINARY_MAPPER);
    config.configureObjectMapper(om);
    BinaryJacksonCoder coder = new BinaryJacksonCoder(workers, validator, om);
    connector.getFilterChain().addLast("codec", new CQProtocolCodecFilter(new ChunkWriter(coder), new ChunkReader(coder, om)));
    connector.setHandler(this);
    connector.getSessionConfig().setAll(config.getCluster().getMina());
    InetSocketAddress address = new InetSocketAddress(config.getCluster().getManagerURL().getHostAddress(), config.getCluster().getPort());
    while (true) {
        try {
            log.info("Trying to connect to {}", address);
            // Try opening a connection (Note: This fails immediately instead of waiting a minute to try and connect)
            ConnectFuture future = connector.connect(address);
            future.awaitUninterruptibly();
            if (future.isConnected()) {
                break;
            }
            future.cancel();
            // Sleep thirty seconds then retry.
            TimeUnit.SECONDS.sleep(30);
        } catch (RuntimeIoException e) {
            log.warn("Failed to connect to " + address, e);
        }
    }
}

Example 5 with Worker

use of com.bakdata.conquery.models.worker.Worker in project conquery by bakdata.

the class ShardNode method sessionOpened.

@Override
public void sessionOpened(IoSession session) throws Exception {
    setLocation(session);
    NetworkSession networkSession = new NetworkSession(session);
    context = new NetworkMessageContext.ShardNodeNetworkContext(jobManager, networkSession, workers, config, validator);
    log.info("Connected to ManagerNode @ {}", session.getRemoteAddress());
    // Authenticate with ManagerNode
    context.send(new AddShardNode());
    for (Worker w : workers.getWorkers().values()) {
        w.setSession(new NetworkSession(session));
        WorkerInformation info = w.getInfo();
        log.info("Sending worker identity '{}'", info.getName());
        networkSession.send(new RegisterWorker(info));
    }
}