Examples with QueueJob dk.dbc.search.solrdocstore.queue.QueueJob used on opensource projects

Example 6 with QueueJob

use of dk.dbc.search.solrdocstore.queue.QueueJob in project solr-document-store by DBCDK.

the class DocProducerIT method deployAndSearch.

private void deployAndSearch(int agencyId, DocProducer docProducer, int expected) throws SolrServerException, IOException {
    JsonNode sourceDoc = docProducer.fetchSourceDoc(new QueueJob(agencyId, "clazzifier", "23645564"));
    SolrInputDocument doc = docProducer.createSolrDocument(sourceDoc);
    String bibliographicShardId = docProducer.bibliographicShardId(sourceDoc);
    docProducer.deleteSolrDocuments(bibliographicShardId, 0);
    docProducer.deploy(doc, 0);
    docProducer.solrClient.commit(true, true);
    QueryResponse response1 = docProducer.solrClient.query(new SolrQuery("*:*"));
    System.out.println("response = " + response1);
    assertEquals(expected, response1.getResults().getNumFound());
}

Example 7 with QueueJob

use of dk.dbc.search.solrdocstore.queue.QueueJob in project solr-document-store by DBCDK.

the class WorkerIT method testQueueWorkerConsumes.

@Test(timeout = 5000L)
public void testQueueWorkerConsumes() throws Exception {
    System.out.println("testQueueWorkerConsumes");
    SolrClient solrClient = SolrApi.makeSolrClient(solrUrl);
    try (Connection connection = dataSource.getConnection()) {
        Requests.load("test1-part1", solrDocStoreUrl);
        long count = solrClient.query(new SolrQuery("*:*")).getResults().getNumFound();
        assertEquals("After delete sorl document count: ", 0, count);
        PreparedQueueSupplier supplier = new QueueSupplier<>(QueueJob.STORAGE_ABSTRACTION).preparedSupplier(connection);
        supplier.enqueue("test", new QueueJob(300000, "clazzifier", "23645564"));
        int maxRuns = 2500 / 50;
        while (solrClient.query(new SolrQuery("*:*")).getResults().getNumFound() == 0) {
            Thread.sleep(50L);
            solrClient.commit(true, true);
            if (maxRuns-- <= 0) {
                break;
            }
        }
        worker.destroy();
        count = solrClient.query(new SolrQuery("*:*")).getResults().getNumFound();
        assertEquals("After dequeue -  sorl document count: ", 3, count);
    }
}

Example 8 with QueueJob

use of dk.dbc.search.solrdocstore.queue.QueueJob in project solr-document-store by DBCDK.

the class QueueAsyncJob method queueAllManifestationsFor.

AsyncJob queueAllManifestationsFor(String queue, boolean deletedToo) {
    return new AsyncJob("Queue-all-for-" + queue) {

        @Override
        public void run(Supplier<Boolean> isCanceled) throws Exception {
            log.debug("Connectiong to database");
            try (Connection connection = dataSource.getConnection()) {
                log.debug("Selecting all records");
                try (Statement stmt = connection.createStatement();
                    ResultSet resultSet = stmt.executeQuery(deletedToo ? ALL_MANIFESTATIONS : ALL_MANIFESTATIONS_NOT_DELETED)) {
                    connection.setAutoCommit(false);
                    boolean shouldCommit = false;
                    EnqueueService<QueueJob> enqueueService = enqueueService(connection, queue);
                    int counter = 0;
                    while (resultSet.next() && !isCanceled.get()) {
                        int i = 0;
                        int agencyId = resultSet.getInt(++i);
                        String classifier = resultSet.getString(++i);
                        String bibliographicRecordId = resultSet.getString(++i);
                        enqueueService.enqueue(new QueueJob(agencyId, classifier, bibliographicRecordId));
                        shouldCommit = true;
                        if (++counter % 2500 == 0) {
                            log.info("Committet: {}", counter);
                            connection.commit();
                            shouldCommit = false;
                        }
                    }
                    if (shouldCommit) {
                        connection.commit();
                        log.info("Committet: {}", counter);
                    }
                }
            } catch (SQLException ex) {
                log.error("Got some sql exception: {}", ex.getMessage());
                log.debug("Got some sql exception: ", ex);
            }
        }
    };
}

Example 9 with QueueJob

use of dk.dbc.search.solrdocstore.queue.QueueJob in project solr-document-store by DBCDK.

the class EnqueueService method enqueue.

public void enqueue(T t, Integer commitWithin) throws SQLException {
    QueueJob job = jobCreator.apply(t, commitWithin);
    for (String queueName : queueNames) {
        log.trace("enqueue to: {}", queueName);
        queueSupplier.enqueue(queueName, job);
    }
}