Examples with JobletRequestQueueKey io.datarouter.joblet.storage.jobletrequestqueue.JobletRequestQueueKey used on opensource projects

Example 1 with JobletRequestQueueKey

use of io.datarouter.joblet.storage.jobletrequestqueue.JobletRequestQueueKey in project datarouter by hotpads.

the class MysqlLockForUpdateJobletRequestSelector method getJobletRequestForProcessing.

@Override
public Optional<JobletRequest> getJobletRequestForProcessing(PhaseTimer timer, JobletType<?> type, String reservedBy) {
    while (true) {
        var mysqlOp = new GetJobletRequest(reservedBy, type, datarouter, jobletRequestDao, mysqlFieldCodecFactory, mysqlSqlFactory, jobletRequestSqlBuilder, jobletService);
        JobletRequest jobletRequest = sessionExecutor.runWithoutRetries(mysqlOp);
        timer.add("GetJobletRequest");
        if (jobletRequest == null) {
            // for back-off
            jobletRequestQueueManager.onJobletRequestMissForAllPriorities(type);
            return Optional.empty();
        }
        if (!jobletRequest.getStatus().isRunning()) {
            // weird flow. it was probably just marked as timedOut, so skip it
            continue;
        }
        var queueKey = new JobletRequestQueueKey(type, jobletRequest.getKey().getPriority());
        datarouterJobletCounters.incQueueHit(queueKey.getQueueName());
        return Optional.of(jobletRequest);
    }
}

Example 2 with JobletRequestQueueKey

use of io.datarouter.joblet.storage.jobletrequestqueue.JobletRequestQueueKey in project datarouter by hotpads.

the class MysqlUpdateAndScanJobletRequestSelector method getJobletRequestForProcessing.

@Override
public Optional<JobletRequest> getJobletRequestForProcessing(PhaseTimer timer, JobletType<?> type, String reservedBy) {
    ReserveJobletRequest mysqlOp = new ReserveJobletRequest(reservedBy, type, datarouter, jobletRequestDao, mysqlSqlFactory, jobletRequestSqlBuilder);
    while (sessionExecutor.runWithoutRetries(mysqlOp)) {
        // returns false if no joblet found
        JobletRequest jobletRequest = jobletRequestDao.getReservedRequest(type, reservedBy, Isolation.readUncommitted);
        if (JobletStatus.CREATED == jobletRequest.getStatus()) {
            jobletRequest.setReservedBy(reservedBy);
            jobletRequest.setReservedAt(System.currentTimeMillis());
            jobletRequest.setStatus(JobletStatus.RUNNING);
            jobletRequestDao.put(jobletRequest);
            return Optional.of(jobletRequest);
        }
        // we got a previously timed-out joblet
        jobletRequest.incrementNumTimeouts();
        if (jobletRequest.getNumTimeouts() <= JobletService.MAX_JOBLET_RETRIES) {
            jobletRequestDao.put(jobletRequest);
            JobletRequestQueueKey queueKey = new JobletRequestQueueKey(type, jobletRequest.getKey().getPriority());
            datarouterJobletCounters.incQueueHit(queueKey.getQueueName());
            return Optional.of(jobletRequest);
        }
        jobletRequest.setStatus(JobletStatus.TIMED_OUT);
        jobletRequestDao.put(jobletRequest);
    // loop around for another
    }
    // for back-off
    jobletRequestQueueManager.onJobletRequestMissForAllPriorities(type);
    return Optional.empty();
}

Example 3 with JobletRequestQueueKey

use of io.datarouter.joblet.storage.jobletrequestqueue.JobletRequestQueueKey in project datarouter by hotpads.

the class JobletService method requeueJobletRequest.

private void requeueJobletRequest(PhaseTimer timer, JobletRequest jobletRequest) {
    if (jobletRequest.getQueueMessageKey() == null) {
        return;
    }
    // rather than ack/put, is there an ack(false) mechanism?
    // maybe https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_ChangeMessageVisibility.html
    ack(jobletRequest);
    timer.add("requeue ack");
    JobletRequestQueueKey queueKey = jobletRequestQueueManager.getQueueKey(jobletRequest);
    jobletQueueDao.getQueue(queueKey).put(jobletRequest, new Config().setPutMethod(PutMethod.UPDATE_OR_BUST));
    timer.add("requeue put");
}

Example 4 with JobletRequestQueueKey

use of io.datarouter.joblet.storage.jobletrequestqueue.JobletRequestQueueKey in project datarouter by hotpads.

the class JobletService method ack.

private void ack(JobletRequest jobletRequest) {
    JobletRequestQueueKey queueKey = jobletRequestQueueManager.getQueueKey(jobletRequest);
    jobletQueueDao.getQueue(queueKey).ack(jobletRequest.getQueueMessageKey());
}

Example 5 with JobletRequestQueueKey

use of io.datarouter.joblet.storage.jobletrequestqueue.JobletRequestQueueKey in project datarouter by hotpads.

the class QueueJobletRequestSelector method getJobletRequestForProcessing.

@Override
public Optional<JobletRequest> getJobletRequestForProcessing(PhaseTimer timer, JobletType<?> type, String reservedBy) {
    for (JobletPriority priority : JobletPriority.values()) {
        JobletRequestQueueKey queueKey = new JobletRequestQueueKey(type, priority);
        if (jobletRequestQueueManager.shouldSkipQueue(queueKey)) {
            datarouterJobletCounters.incQueueSkip(queueKey.getQueueName());
            continue;
        }
        // set timeout to 0 so we return immediately. processor threads can do the waiting
        Config config = new Config().setTimeout(Duration.ofMillis(0)).setVisibilityTimeoutMs(datarouterJobletSettingRoot.jobletTimeout.get().toMillis());
        logger.info("jobletType={} queue={}", type, jobletQueueDao.getQueue(queueKey));
        QueueMessage<JobletRequestKey, JobletRequest> message = jobletQueueDao.getQueue(queueKey).peek(config);
        timer.add("peek");
        if (message == null) {
            jobletRequestQueueManager.onJobletRequestQueueMiss(queueKey);
            continue;
        }
        datarouterJobletCounters.incQueueHit(queueKey.getQueueName());
        JobletRequest jobletRequest = message.getDatabean();
        boolean existsInDb = jobletRequestDao.exists(jobletRequest.getKey());
        timer.add("check exists");
        if (!existsInDb) {
            logger.warn("draining non-existent JobletRequest without processing jobletRequest={}", jobletRequest);
            datarouterJobletCounters.ignoredRequestMissingFromDb(type);
            jobletQueueDao.getQueue(queueKey).ack(message.getKey());
            timer.add("ack missing request");
            continue;
        }
        jobletRequest.setQueueMessageKey(message.getKey());
        jobletRequest.setReservedBy(reservedBy);
        jobletRequest.setReservedAt(System.currentTimeMillis());
        jobletRequest.setStatus(JobletStatus.RUNNING);
        jobletRequestDao.put(jobletRequest);
        if (!jobletRequest.getRestartable()) {
            // don't let SQS give this joblet out again
            jobletQueueDao.getQueue(queueKey).ack(message.getKey());
            timer.add("ack non-restartable");
        }
        return Optional.of(jobletRequest);
    }
    return Optional.empty();
}