Examples with QueueJob dk.dbc.search.solrdocstore.queue.QueueJob used on opensource projects

Example 1 with QueueJob

use of dk.dbc.search.solrdocstore.queue.QueueJob in project solr-document-store by DBCDK.

the class Worker method makeWorker.

public JobConsumer<QueueJob> makeWorker() {
    return new JobConsumer<QueueJob>() {

        @Override
        public void accept(Connection connection, QueueJob job, JobMetaData metaData) throws FatalQueueError, NonFatalQueueError, PostponedNonFatalQueueError {
            log.info("job = {}, metadata = {}", job, metaData);
            try {
                JsonNode sourceDoc = docProducer.fetchSourceDoc(job);
                SolrInputDocument solrDocument = docProducer.createSolrDocument(sourceDoc);
                String bibliographicShardId = docProducer.bibliographicShardId(sourceDoc);
                docProducer.deleteSolrDocuments(bibliographicShardId, job.getCommitwithin());
                docProducer.deploy(solrDocument, job.getCommitwithin());
            } catch (IOException ex) {
                throw new NonFatalQueueError(ex);
            } catch (SolrServerException ex) {
                throw new FatalQueueError(ex);
            }
        }
    };
}

Example 2 with QueueJob

use of dk.dbc.search.solrdocstore.queue.QueueJob in project solr-document-store by DBCDK.

the class DocProducerTest method test.

/**
 * Test of init method, of class DocProducer.
 */
@Test
public void test() throws Exception {
    System.out.println("test");
    DocProducer docProducer = new DocProducer() {

        @Override
        public JsonNode fetchSourceDoc(QueueJob job) throws IOException {
            String file = "DocProducerTest/" + job.getAgencyId() + "-" + job.getBibliographicRecordId() + ".json";
            try (InputStream stream = DocProducerTest.class.getClassLoader().getResourceAsStream(file)) {
                return OBJECT_MAPPER.readTree(stream);
            }
        }
    };
    docProducer.solrFields = SolrFieldsTest.newSolrFields("schema.xml", "http://some.crazy.host/with/a/strange/path");
    JsonNode node = docProducer.fetchSourceDoc(new QueueJob(300101, "clazzifier", "23645564"));
    System.out.println("node = " + node);
    assertFalse(docProducer.isDeleted(node));
    SolrInputDocument document = docProducer.inputDocument(node);
    System.out.println("document = " + document);
    String xml = ClientUtils.toXML(document);
    assertTrue(document.containsKey("dkcclterm.po"));
    assertFalse(document.containsKey("unknown.field"));
    assertEquals(3, document.get("rec.holdingsAgencyId").getValues().size());
    assertTrue(document.get("rec.holdingsAgencyId").getValues().contains("300101"));
    assertTrue(document.get("rec.holdingsAgencyId").getValues().contains("300102"));
    assertTrue(document.get("rec.holdingsAgencyId").getValues().contains("300104"));
    assertTrue(document.get("rec.repositoryId").getValues().contains("300101-katalog:23645564"));
    String linkId = document.getField("rec.childDocId").getValue().toString();
    for (SolrInputDocument childDocument : document.getChildDocuments()) {
        assertEquals(linkId, childDocument.getField("parentDocId").getValue().toString());
    }
    System.out.println("OK");
}

Example 3 with QueueJob

use of dk.dbc.search.solrdocstore.queue.QueueJob in project solr-document-store by DBCDK.

the class QueueAsyncJob method queueErrorRequeueJobs.

AsyncJob queueErrorRequeueJobs(String consumer, String pattern) {
    String like = makeLike(pattern);
    return new AsyncJob("Requeue-errors-matching-" + pattern) {

        @Override
        public void run(Supplier<Boolean> isCanceled) throws Exception {
            log.debug("Connectiong to database");
            try (Connection connection = dataSource.getConnection();
                PreparedStatement stmt = makeErrorQuery(connection, consumer, like);
                PreparedStatement del = connection.prepareStatement("DELETE FROM queue_error WHERE ctid = ?")) {
                connection.setAutoCommit(false);
                boolean shouldCommit = false;
                int counter = 0;
                HashMap<String, EnqueueService<QueueJob>> enqueueServices = new HashMap<>();
                try (ResultSet resultSet = stmt.executeQuery()) {
                    while (!isCanceled.get() && resultSet.next()) {
                        ErrorEntry err = new ErrorEntry(resultSet);
                        log.info("Requeue: {}-{}:{} | {}:{} | {} | {}", err.getAgencyId(), err.getClassifier(), err.getBibliographicRecordId(), err.getQueued(), err.getFailedAt(), err.getJobConsumer(), err.getDiag());
                        EnqueueService<QueueJob> enqueueService = enqueueServices.computeIfAbsent(err.getJobConsumer(), c -> enqueueService(connection, c));
                        enqueueService.enqueue(err.toQueueJob());
                        shouldCommit = true;
                        del.setObject(1, err.getCtid());
                        del.executeUpdate();
                        if (++counter % 2500 == 0) {
                            connection.commit();
                            shouldCommit = false;
                        }
                    }
                    if (shouldCommit) {
                        connection.commit();
                    }
                }
            }
        }
    };
}

Example 4 with QueueJob

use of dk.dbc.search.solrdocstore.queue.QueueJob in project solr-document-store by DBCDK.

the class DocTest method format.

@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("format/{agencyId : \\d+}/{classifier}/{bibliographicRecordId : .*$}")
public Response format(@PathParam("agencyId") int agencyId, @PathParam("classifier") String classifier, @PathParam("bibliographicRecordId") String bibliographicRecordId) throws InterruptedException, ExecutionException, IOException {
    log.debug("agencyId = {}; bibliographicRecordId = {}", agencyId, bibliographicRecordId);
    try {
        JsonNode node = docProducer.fetchSourceDoc(new QueueJob(agencyId, classifier, bibliographicRecordId));
        boolean deleted = docProducer.isDeleted(node);
        if (deleted) {
            return Response.ok(false).build();
        }
        SolrInputDocument document = docProducer.inputDocument(node);
        String xml = ClientUtils.toXML(document);
        return Response.ok(xml, MediaType.APPLICATION_XML_TYPE).build();
    } catch (IOException ex) {
        log.error("Exception: {}", ex.getMessage());
        log.debug("Exception:", ex);
        return Response.ok(ex.getMessage(), MediaType.TEXT_PLAIN).build();
    }
}

Example 5 with QueueJob

use of dk.dbc.search.solrdocstore.queue.QueueJob in project solr-document-store by DBCDK.

the class DocTest method deploy.

@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("deploy/{agencyId : \\d+}/{classifier}/{bibliographicRecordId : .*$}")
public Response deploy(@PathParam("agencyId") int agencyId, @PathParam("classifier") String classifier, @PathParam("bibliographicRecordId") String bibliographicRecordId, @QueryParam("commitWithin") Integer commitWithin) throws InterruptedException, ExecutionException, IOException {
    try {
        JsonNode sourceDoc = docProducer.fetchSourceDoc(new QueueJob(agencyId, classifier, bibliographicRecordId));
        SolrInputDocument doc = docProducer.createSolrDocument(sourceDoc);
        String bibliographicShardId = docProducer.bibliographicShardId(sourceDoc);
        docProducer.deleteSolrDocuments(bibliographicShardId, 0);
        docProducer.deploy(doc, commitWithin);
        return Response.ok("{\"ok\":true}", MediaType.APPLICATION_XML_TYPE).build();
    } catch (SolrServerException | IOException ex) {
        log.error("Exception: {}", ex.getMessage());
        log.debug("Exception:", ex);
        return Response.ok(ex.getMessage()).build();
    }
}