Examples with AsyncJob dk.dbc.search.solrdocstore.asyncjob.AsyncJob used on opensource projects

Example 1 with AsyncJob

use of dk.dbc.search.solrdocstore.asyncjob.AsyncJob in project solr-document-store by DBCDK.

the class QueueAsyncJob method queueErrorListJobs.

AsyncJob queueErrorListJobs(String consumer, String pattern) {
    String like = makeLike(pattern);
    return new AsyncJob("List-errors-matching-" + pattern) {

        @Override
        public String getLogPattern() {
            return "";
        }

        @Override
        public void run(Supplier<Boolean> isCanceled) throws Exception {
            log.debug("Connectiong to database");
            try (Connection connection = dataSource.getConnection();
                PreparedStatement stmt = makeErrorQuery(connection, consumer, like)) {
                log.info("========= MATCHES BEGIN ========");
                try (ResultSet resultSet = stmt.executeQuery()) {
                    while (!isCanceled.get() && resultSet.next()) {
                        ErrorEntry err = new ErrorEntry(resultSet);
                        log.info("{}-{}:{} | {}:{} | {} | {}", err.getAgencyId(), err.getClassifier(), err.getBibliographicRecordId(), err.getQueued(), err.getFailedAt(), err.getJobConsumer(), err.getDiag());
                    }
                }
                log.info("========= MATCHES END ========");
            }
        }
    };
}

Example 2 with AsyncJob

use of dk.dbc.search.solrdocstore.asyncjob.AsyncJob in project solr-document-store by DBCDK.

the class QueueAsyncJob method queueErrorRequeueJobs.

AsyncJob queueErrorRequeueJobs(String consumer, String pattern) {
    String like = makeLike(pattern);
    return new AsyncJob("Requeue-errors-matching-" + pattern) {

        @Override
        public void run(Supplier<Boolean> isCanceled) throws Exception {
            log.debug("Connectiong to database");
            try (Connection connection = dataSource.getConnection();
                PreparedStatement stmt = makeErrorQuery(connection, consumer, like);
                PreparedStatement del = connection.prepareStatement("DELETE FROM queue_error WHERE ctid = ?")) {
                connection.setAutoCommit(false);
                boolean shouldCommit = false;
                int counter = 0;
                HashMap<String, EnqueueService<QueueJob>> enqueueServices = new HashMap<>();
                try (ResultSet resultSet = stmt.executeQuery()) {
                    while (!isCanceled.get() && resultSet.next()) {
                        ErrorEntry err = new ErrorEntry(resultSet);
                        log.info("Requeue: {}-{}:{} | {}:{} | {} | {}", err.getAgencyId(), err.getClassifier(), err.getBibliographicRecordId(), err.getQueued(), err.getFailedAt(), err.getJobConsumer(), err.getDiag());
                        EnqueueService<QueueJob> enqueueService = enqueueServices.computeIfAbsent(err.getJobConsumer(), c -> enqueueService(connection, c));
                        enqueueService.enqueue(err.toQueueJob());
                        shouldCommit = true;
                        del.setObject(1, err.getCtid());
                        del.executeUpdate();
                        if (++counter % 2500 == 0) {
                            connection.commit();
                            shouldCommit = false;
                        }
                    }
                    if (shouldCommit) {
                        connection.commit();
                    }
                }
            }
        }
    };
}

Example 3 with AsyncJob

use of dk.dbc.search.solrdocstore.asyncjob.AsyncJob in project solr-document-store by DBCDK.

the class QueueAsyncJob method queueAllManifestationsFor.

AsyncJob queueAllManifestationsFor(String queue, boolean deletedToo) {
    return new AsyncJob("Queue-all-for-" + queue) {

        @Override
        public void run(Supplier<Boolean> isCanceled) throws Exception {
            log.debug("Connectiong to database");
            try (Connection connection = dataSource.getConnection()) {
                log.debug("Selecting all records");
                try (Statement stmt = connection.createStatement();
                    ResultSet resultSet = stmt.executeQuery(deletedToo ? ALL_MANIFESTATIONS : ALL_MANIFESTATIONS_NOT_DELETED)) {
                    connection.setAutoCommit(false);
                    boolean shouldCommit = false;
                    EnqueueService<QueueJob> enqueueService = enqueueService(connection, queue);
                    int counter = 0;
                    while (resultSet.next() && !isCanceled.get()) {
                        int i = 0;
                        int agencyId = resultSet.getInt(++i);
                        String classifier = resultSet.getString(++i);
                        String bibliographicRecordId = resultSet.getString(++i);
                        enqueueService.enqueue(new QueueJob(agencyId, classifier, bibliographicRecordId));
                        shouldCommit = true;
                        if (++counter % 2500 == 0) {
                            log.info("Committet: {}", counter);
                            connection.commit();
                            shouldCommit = false;
                        }
                    }
                    if (shouldCommit) {
                        connection.commit();
                        log.info("Committet: {}", counter);
                    }
                }
            } catch (SQLException ex) {
                log.error("Got some sql exception: {}", ex.getMessage());
                log.debug("Got some sql exception: ", ex);
            }
        }
    };
}

Example 4 with AsyncJob

use of dk.dbc.search.solrdocstore.asyncjob.AsyncJob in project solr-document-store by DBCDK.

the class QueueAsyncJob method queueErrorDeleteJobs.

AsyncJob queueErrorDeleteJobs(String consumer, String pattern) {
    String like = makeLike(pattern);
    return new AsyncJob("Delete-errors-matching-" + pattern) {

        @Override
        public void run(Supplier<Boolean> isCanceled) throws Exception {
            log.debug("Connectiong to database");
            try (Connection connection = dataSource.getConnection();
                PreparedStatement stmt = makeErrorQuery(connection, consumer, like);
                PreparedStatement del = connection.prepareStatement("DELETE FROM queue_error WHERE ctid = ?")) {
                connection.setAutoCommit(false);
                boolean shouldCommit = false;
                int counter = 0;
                try (ResultSet resultSet = stmt.executeQuery()) {
                    while (!isCanceled.get() && resultSet.next()) {
                        ErrorEntry err = new ErrorEntry(resultSet);
                        log.info("Remove: {}-{}:{} | {}:{} | {} | {}", err.getAgencyId(), err.getClassifier(), err.getBibliographicRecordId(), err.getQueued(), err.getFailedAt(), err.getJobConsumer(), err.getDiag());
                        del.setObject(1, err.getCtid());
                        if (del.executeUpdate() != 0) {
                            shouldCommit = true;
                        }
                        if (++counter % 2500 == 0) {
                            connection.commit();
                            shouldCommit = false;
                        }
                    }
                    if (shouldCommit) {
                        connection.commit();
                    }
                }
            }
        }
    };
}