Examples with TableSample io.datarouter.nodewatch.storage.tablesample.TableSample used on opensource projects

Example 1 with TableSample

use of io.datarouter.nodewatch.storage.tablesample.TableSample in project datarouter by hotpads.

the class TableSpanSampler method makeSample.

/*---------------- sample handling -----------------*/
// helpful to enforce the key is PK vs TableSampleKey
private TableSample makeSample(String reason, PK pk, Instant forceCreatedAt, boolean markInterrupted, boolean isLastSpan) {
    String logCreatedAt = Optional.ofNullable(forceCreatedAt).map(Object::toString).orElse("now");
    String log = "makeSample reason=" + reason + ", createdAt=" + logCreatedAt + ", markInterrupted=" + markInterrupted + ", isLastSpan=" + isLastSpan + ", " + this;
    logger.info(log);
    Date sampleDateCreated = Optional.ofNullable(forceCreatedAt).map(Date::from).orElseGet(Date::new);
    var sample = new TableSample(nodeNames, pk.getFields(), numSinceLastMarker, sampleDateCreated, getLatestSpanCountTime().toMillis(), markInterrupted, isLastSpan);
    return sample;
}

Example 2 with TableSample

use of io.datarouter.nodewatch.storage.tablesample.TableSample in project datarouter by hotpads.

the class TableSpanSampler method handleMovedEndOfTable.

private void handleMovedEndOfTable() {
    deleteEndSample("moved");
    TableSample sample = makeSample("movedEnd", latestPk, null, false, true);
    putAndKeepSample(sample);
}

Example 3 with TableSample

use of io.datarouter.nodewatch.storage.tablesample.TableSample in project datarouter by hotpads.

the class TableSpanSampler method handleEndOfIntermediateSpanOnInterrupt.

private void handleEndOfIntermediateSpanOnInterrupt() {
    TableSample sample = makeSample("interruptedIntermediate", pkRange.getEnd(), createdAt, true, false);
    updateEndSampleOnInterrupt(sample);
    putAndKeepSample(sample);
}

Example 4 with TableSample

use of io.datarouter.nodewatch.storage.tablesample.TableSample in project datarouter by hotpads.

the class TableSpanSamplerJoblet method process.

@Override
public void process() {
    TableSample dbSample = tableSampleDao.get(params.endSample.getKey());
    if (dbSample == null) {
        logger.warn("aborting because dbSample missing {}", params.endSample);
        return;
    }
    if (ObjectTool.notEquals(params.samplerId, dbSample.getSamplerId())) {
        logger.warn("aborting because wrong samplerId={}, {}", params.samplerId, params.endSample);
        return;
    }
    // TODO make expiration a generic joblet feature
    Duration age = jobletRequest.getKey().getAge();
    if (ComparableTool.gt(age, TableSample.MAX_TIME_IN_QUEUE)) {
        logger.warn("aborting expired joblet {} with age {}", jobletRequest, age);
        datarouterJobletCounters.incNumJobletsExpired(1);
        datarouterJobletCounters.incNumJobletsExpired(JOBLET_TYPE, 1);
        return;
    // the joblet creator will quickly create another one based on the stale dateScheduled
    }
    PhysicalNode<?, ?, ?> physicalNode = datarouterNodes.getPhysicalNodeForClientAndTable(params.nodeNames.getClientName(), params.nodeNames.getTableName());
    SortedStorageReaderNode<?, ?, ?> node = (PhysicalSortedStorageReaderNode<?, ?, ?>) physicalNode;
    Objects.requireNonNull(node, "node not found for " + params.nodeNames);
    // TODO replace strings with more formal client detection
    boolean clientSupportsOffsetting = physicalNode.getClientType().supportsOffsetSampling();
    Instant deadline = Instant.now().plus(MAX_RUNNING_TIME);
    samples = new TableSpanSampler<>(node, tableSampleDao, params.samplerId, params.nodeNames, params.startSampleKey, params.endSample, params.sampleEveryN, clientSupportsOffsetting && nodewatchSettingRoot.enableOffsetting.get(), params.batchSize, Instant.ofEpochMilli(params.createdTimeMs), params.scanUntilEnd, deadline).call();
}

Example 5 with TableSample

use of io.datarouter.nodewatch.storage.tablesample.TableSample in project datarouter by hotpads.

the class TableSpanSamplerJobletCreator method createJoblets.

public List<JobletPackage> createJoblets() {
    var timer = new PhaseTimer(nodeNames.toString());
    var existingSamples = new PeekingIterator<>(tableSampleDao.streamForNode(nodeNames).iterator());
    if (!existingSamples.hasNext()) {
        handleNoExistingSamples();
    } else {
        TableSample previous = null;
        while (existingSamples.hasNext()) {
            TableSample current = existingSamples.next();
            ++numExisting;
            boolean isLastSample = !existingSamples.hasNext();
            repairLastSampleIfNecessary(current, isLastSample);
            TableSample next = existingSamples.peek();
            if (tryMergeFirstIntoSecond(current, next)) {
                continue;
            }
            considerCreatingJoblet(previous, current, isLastSample);
            resetInterruptedFlagIfNecessary(previous);
            previous = current;
        }
    }
    // TODO check if first sample was considered or merged
    logSummary();
    timer.add("scan");
    if (jobletPackages.size() > 0 && submitJoblets) {
        jobletService.submitJobletPackages(jobletPackages);
    }
    timer.add("submitJoblets");
    logger.debug("{}", timer);
    if (timer.getElapsedTimeBetweenFirstAndLastEvent() > Duration.ofSeconds(5).toMillis()) {
        logger.warn("{}", timer);
    }
    return jobletPackages;
}