Examples with JobletPriority io.datarouter.joblet.enums.JobletPriority used on opensource projects

Example 1 with JobletPriority

use of io.datarouter.joblet.enums.JobletPriority in project datarouter by hotpads.

the class TableSpanSamplerJobletCreator method considerCreatingJoblet.

private void considerCreatingJoblet(TableSample start, TableSample end, boolean isLastSample) {
    numConsidered++;
    Pair<Boolean, JobletPriority> shouldCountAndPriority = shouldCount(end, isLastSample);
    boolean shouldCount = shouldCountAndPriority.getLeft();
    JobletPriority jobletPriority = shouldCountAndPriority.getRight();
    if (!shouldCount) {
        return;
    }
    long samplerId = RandomTool.nextPositiveLong();
    numIncluded++;
    jobletPackages.add(createJobletPackage(jobletPriority, start, end, samplerId, isLastSample));
    if (jobletPackages.size() > BATCH_SIZE && submitJoblets) {
        jobletService.submitJobletPackages(jobletPackages);
        // TODO clearing violates the return value that integration tests are expecting
        jobletPackages.clear();
    }
    end.setScheduleFields(samplerId, new Date());
    tableSampleDao.put(end);
}

Example 2 with JobletPriority

use of io.datarouter.joblet.enums.JobletPriority in project datarouter by hotpads.

the class QueueJobletRequestSelector method getJobletRequestForProcessing.

@Override
public Optional<JobletRequest> getJobletRequestForProcessing(PhaseTimer timer, JobletType<?> type, String reservedBy) {
    for (JobletPriority priority : JobletPriority.values()) {
        JobletRequestQueueKey queueKey = new JobletRequestQueueKey(type, priority);
        if (jobletRequestQueueManager.shouldSkipQueue(queueKey)) {
            datarouterJobletCounters.incQueueSkip(queueKey.getQueueName());
            continue;
        }
        // set timeout to 0 so we return immediately. processor threads can do the waiting
        Config config = new Config().setTimeout(Duration.ofMillis(0)).setVisibilityTimeoutMs(datarouterJobletSettingRoot.jobletTimeout.get().toMillis());
        logger.info("jobletType={} queue={}", type, jobletQueueDao.getQueue(queueKey));
        QueueMessage<JobletRequestKey, JobletRequest> message = jobletQueueDao.getQueue(queueKey).peek(config);
        timer.add("peek");
        if (message == null) {
            jobletRequestQueueManager.onJobletRequestQueueMiss(queueKey);
            continue;
        }
        datarouterJobletCounters.incQueueHit(queueKey.getQueueName());
        JobletRequest jobletRequest = message.getDatabean();
        boolean existsInDb = jobletRequestDao.exists(jobletRequest.getKey());
        timer.add("check exists");
        if (!existsInDb) {
            logger.warn("draining non-existent JobletRequest without processing jobletRequest={}", jobletRequest);
            datarouterJobletCounters.ignoredRequestMissingFromDb(type);
            jobletQueueDao.getQueue(queueKey).ack(message.getKey());
            timer.add("ack missing request");
            continue;
        }
        jobletRequest.setQueueMessageKey(message.getKey());
        jobletRequest.setReservedBy(reservedBy);
        jobletRequest.setReservedAt(System.currentTimeMillis());
        jobletRequest.setStatus(JobletStatus.RUNNING);
        jobletRequestDao.put(jobletRequest);
        if (!jobletRequest.getRestartable()) {
            // don't let SQS give this joblet out again
            jobletQueueDao.getQueue(queueKey).ack(message.getKey());
            timer.add("ack non-restartable");
        }
        return Optional.of(jobletRequest);
    }
    return Optional.empty();
}

Example 3 with JobletPriority

use of io.datarouter.joblet.enums.JobletPriority in project datarouter by hotpads.

the class SleepingJobletHandler method createSleepingJoblets.

@Handler(defaultHandler = true)
private Mav createSleepingJoblets(@Param(P_numJoblets) OptionalInteger numJoblets, @Param(P_sleepMs) OptionalLong sleepMs, @Param(P_executionOrder) OptionalInteger executionOrder, @Param(P_includeFailures) OptionalBoolean includeFailures, @Param(P_failEveryN) OptionalInteger failEveryN, @Param(P_submitAction) OptionalString submitAction) {
    var form = new HtmlForm().withMethod("post");
    form.addTextField().withDisplay("Number of Joblets").withName(P_numJoblets).withPlaceholder("1000").withValue(numJoblets.map(Object::toString).orElse(1000 + ""));
    form.addTextField().withDisplay("Sleep Millis").withName(P_sleepMs).withPlaceholder("1000").withValue(sleepMs.map(Object::toString).orElse(1000 + ""));
    form.addTextField().withDisplay("Execution Order").withName(P_executionOrder).withPlaceholder(JobletPriority.DEFAULT.getExecutionOrder() + "").withValue(executionOrder.map(Object::toString).orElse(JobletPriority.DEFAULT.getExecutionOrder() + ""));
    form.addCheckboxField().withDisplay("Include Failures").withName(P_includeFailures).withChecked(includeFailures.orElse(true));
    form.addTextField().withDisplay("Fail Every N").withName(P_failEveryN).withPlaceholder(100 + "").withValue(failEveryN.map(Object::toString).orElse(100 + ""));
    form.addButton().withDisplay("Create Joblets").withValue("anything");
    if (submitAction.isEmpty() || form.hasErrors()) {
        return pageFactory.startBuilder(request).withTitle(TITLE).withContent(makeContent(form)).buildMav();
    }
    JobletPriority priority = JobletPriority.fromExecutionOrder(executionOrder.get());
    List<JobletPackage> jobletPackages = new ArrayList<>();
    for (int i = 0; i < numJoblets.get(); ++i) {
        int numFailuresForThisJoblet = 0;
        if (includeFailures.orElse(false)) {
            boolean failThisJoblet = i % failEveryN.orElse(10) == 0;
            if (failThisJoblet) {
                // +3 to see if it causes a problem
                numFailuresForThisJoblet = JobletRequest.MAX_FAILURES + 3;
            }
        }
        SleepingJobletParams params = new SleepingJobletParams(String.valueOf(i), sleepMs.get(), numFailuresForThisJoblet);
        // specify this so joblets execute in precise order
        int batchSequence = i;
        JobletPackage jobletPackage = JobletPackage.createDetailed(SleepingJoblet.JOBLET_TYPE, priority, Instant.now(), batchSequence, true, null, null, params);
        jobletPackages.add(jobletPackage);
    }
    jobletService.submitJobletPackages(jobletPackages);
    return pageFactory.message(request, String.format("created %s @%s ms each", numJoblets.get(), sleepMs.get()));
}