Examples with Config io.datarouter.storage.config.Config used on opensource projects

Example 1 with Config

use of io.datarouter.storage.config.Config in project datarouter by hotpads.

the class JobletService method requeueJobletRequest.

private void requeueJobletRequest(PhaseTimer timer, JobletRequest jobletRequest) {
    if (jobletRequest.getQueueMessageKey() == null) {
        return;
    }
    // rather than ack/put, is there an ack(false) mechanism?
    // maybe https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_ChangeMessageVisibility.html
    ack(jobletRequest);
    timer.add("requeue ack");
    JobletRequestQueueKey queueKey = jobletRequestQueueManager.getQueueKey(jobletRequest);
    jobletQueueDao.getQueue(queueKey).put(jobletRequest, new Config().setPutMethod(PutMethod.UPDATE_OR_BUST));
    timer.add("requeue put");
}

Example 2 with Config

use of io.datarouter.storage.config.Config in project datarouter by hotpads.

the class QueueJobletRequestSelector method getJobletRequestForProcessing.

@Override
public Optional<JobletRequest> getJobletRequestForProcessing(PhaseTimer timer, JobletType<?> type, String reservedBy) {
    for (JobletPriority priority : JobletPriority.values()) {
        JobletRequestQueueKey queueKey = new JobletRequestQueueKey(type, priority);
        if (jobletRequestQueueManager.shouldSkipQueue(queueKey)) {
            datarouterJobletCounters.incQueueSkip(queueKey.getQueueName());
            continue;
        }
        // set timeout to 0 so we return immediately. processor threads can do the waiting
        Config config = new Config().setTimeout(Duration.ofMillis(0)).setVisibilityTimeoutMs(datarouterJobletSettingRoot.jobletTimeout.get().toMillis());
        logger.info("jobletType={} queue={}", type, jobletQueueDao.getQueue(queueKey));
        QueueMessage<JobletRequestKey, JobletRequest> message = jobletQueueDao.getQueue(queueKey).peek(config);
        timer.add("peek");
        if (message == null) {
            jobletRequestQueueManager.onJobletRequestQueueMiss(queueKey);
            continue;
        }
        datarouterJobletCounters.incQueueHit(queueKey.getQueueName());
        JobletRequest jobletRequest = message.getDatabean();
        boolean existsInDb = jobletRequestDao.exists(jobletRequest.getKey());
        timer.add("check exists");
        if (!existsInDb) {
            logger.warn("draining non-existent JobletRequest without processing jobletRequest={}", jobletRequest);
            datarouterJobletCounters.ignoredRequestMissingFromDb(type);
            jobletQueueDao.getQueue(queueKey).ack(message.getKey());
            timer.add("ack missing request");
            continue;
        }
        jobletRequest.setQueueMessageKey(message.getKey());
        jobletRequest.setReservedBy(reservedBy);
        jobletRequest.setReservedAt(System.currentTimeMillis());
        jobletRequest.setStatus(JobletStatus.RUNNING);
        jobletRequestDao.put(jobletRequest);
        if (!jobletRequest.getRestartable()) {
            // don't let SQS give this joblet out again
            jobletQueueDao.getQueue(queueKey).ack(message.getKey());
            timer.add("ack non-restartable");
        }
        return Optional.of(jobletRequest);
    }
    return Optional.empty();
}

Example 3 with Config

use of io.datarouter.storage.config.Config in project datarouter by hotpads.

the class DatarouterJobletRequestDao method scanTypePriority.

public Scanner<JobletRequest> scanTypePriority(JobletType<?> type, JobletPriority priority, boolean anyDelay) {
    var prefix = JobletRequestKey.create(type, priority.getExecutionOrder(), null, null);
    var config = new Config().setAnyDelay(anyDelay);
    return node.scanWithPrefix(prefix, config);
}

Example 4 with Config

use of io.datarouter.storage.config.Config in project datarouter by hotpads.

the class MapCachingMapStorageReaderNode method get.

@Override
public D get(PK key, Config config) {
    if (!useCache(config)) {
        return backingNode.get(key, config);
    }
    Config effectiveCachingNodeConfig = getEffectiveCachingNodeConfig(config);
    D cachedObject;
    try {
        updateLastAttemptedContact();
        cachedObject = cachingNode.get(key, effectiveCachingNodeConfig);
        updateLastContact();
    } catch (Exception e) {
        countExceptions();
        return backingNode.get(key, config);
    }
    if (cachedObject != null) {
        countHits();
        return cachedObject;
    }
    D realObject = backingNode.get(key, config);
    if (realObject != null) {
        countMisses();
        if (cacheReads) {
            try {
                updateLastAttemptedContact();
                cachingNode.put(realObject, effectiveCachingNodeConfig);
                updateLastContact();
            } catch (Exception e) {
                countExceptions();
            }
        }
    }
    return realObject;
}

Example 5 with Config

use of io.datarouter.storage.config.Config in project datarouter by hotpads.

the class MapCachingMapStorageWriterMixin method putMulti.

@Override
public void putMulti(Collection<D> databeans, Config config) {
    if (databeans == null || databeans.isEmpty()) {
        return;
    }
    if (BaseMapCachingNode.useCache(config)) {
        target.updateLastAttemptedContact();
        Config effectiveCachingNodeConfig = MapCachingMapStorageReaderNode.getEffectiveCachingNodeConfig(config);
        if (cacheWrites) {
            target.getCachingNode().putMulti(databeans, effectiveCachingNodeConfig);
        } else {
            // TODO check config for ignoring caching
            Scanner.of(databeans).map(Databean::getKey).flush(keys -> target.getCachingNode().deleteMulti(keys, effectiveCachingNodeConfig));
        }
        target.updateLastContact();
    }
    try {
        target.getBackingNode().putMulti(databeans, config);
    } catch (Exception ex) {
        if (BaseMapCachingNode.useCache(config)) {
            target.updateLastAttemptedContact();
            Config effectiveCachingNodeConfig = MapCachingMapStorageReaderNode.getEffectiveCachingNodeConfig(config);
            Scanner.of(databeans).map(Databean::getKey).flush(keys -> target.getCachingNode().deleteMulti(keys, effectiveCachingNodeConfig));
            target.updateLastContact();
        }
        throw ex;
    }
}