Examples with Twin io.datarouter.util.tuple.Twin used on opensource projects

Example 1 with Twin

use of io.datarouter.util.tuple.Twin in project datarouter by hotpads.

the class DatarouterSnapshotEntryHandler method buildTable.

private ContainerTag<?> buildTable(SnapshotKey snapshotKey, long id) {
    SnapshotGroup group = groups.getGroup(snapshotKey.groupId);
    var reader = new SnapshotIdReader(snapshotKey, groups);
    SnapshotRecord record = reader.getRecord(id);
    SnapshotRecordStringDecoder decoder = ReflectionTool.create(group.getSnapshotEntryDecoderClass());
    SnapshotRecordStrings decoded = decoder.decode(record);
    List<Twin<String>> rows = new ArrayList<>();
    rows.add(new Twin<>("id", Long.toString(record.id)));
    rows.add(new Twin<>(decoder.keyName(), decoded.key));
    rows.add(new Twin<>(decoder.valueName(), decoded.value));
    IntStream.range(0, decoded.columnValues.size()).mapToObj(column -> new Twin<>(decoder.columnValueName(column), decoded.columnValues.get(column))).forEach(rows::add);
    var table = new J2HtmlTable<Twin<String>>().withClasses("sortable table table-sm table-striped my-4 border").withColumn("field", twin -> twin.getLeft()).withColumn("value", twin -> twin.getRight()).build(rows);
    return table;
}

Example 2 with Twin

use of io.datarouter.util.tuple.Twin in project datarouter by hotpads.

the class TableSizeMonitoringService method getAboveThresholdLists.

public Twin<List<CountStat>> getAboveThresholdLists() {
    List<CountStat> aboveThresholdList = new ArrayList<>();
    List<CountStat> abovePercentageList = new ArrayList<>();
    for (PhysicalNode<?, ?, ?> node : datarouterNodes.getWritableNodes(clients.getClientIds())) {
        ClientTableEntityPrefixNameWrapper nodeNames = new ClientTableEntityPrefixNameWrapper(node);
        String tableName = nodeNames.getTableName();
        String clientName = nodeNames.getClientName();
        NodewatchConfiguration nodeConfig = null;
        Long threshold = null;
        boolean enablePercentChangeAlert = true;
        boolean enableThresholdAlert = true;
        nodeConfig = tableConfigurationService.getTableConfigMap().get(nodeNames);
        if (nodeConfig != null) {
            threshold = nodeConfig.maxThreshold;
            enablePercentChangeAlert = nodeConfig.enablePercentageAlert;
            enableThresholdAlert = nodeConfig.enableThresholdAlert;
        }
        // continue if the nodeConfig isCountable is set to false
        if (nodeConfig != null && !nodeConfig.isCountable) {
            continue;
        }
        List<TableCount> tableCountRecords = tableCountDao.getForTable(clientName, tableName);
        if (tableCountRecords.size() < 2) {
            continue;
        }
        Collections.sort(tableCountRecords, new TableCount.TableCountLatestEntryComparator());
        TableCount latest = tableCountRecords.get(0);
        TableCount previous = tableCountRecords.get(1);
        if (previous.getNumRows() == 0) {
            continue;
        }
        // skip if the table has records less than the count_threshold
        if (smallEnoughToIgnore(latest.getNumRows()) && smallEnoughToIgnore(previous.getNumRows())) {
            continue;
        }
        if (enableThresholdAlert) {
            Optional<TableSizeAlertThreshold> thresholdEntry = tableSizeAlertThresholdDao.find(new TableSizeAlertThresholdKey(clientName, tableName));
            // override manual thresholdEntry if exists
            if (thresholdEntry.isPresent() && thresholdEntry.get().getMaxRows() > 0) {
                threshold = thresholdEntry.get().getMaxRows();
            }
            // check if node numRows exceeds threshold
            if (threshold != null && latest.getNumRows() >= threshold) {
                aboveThresholdList.add(calculateStats(latest, threshold));
            }
        }
        if (enablePercentChangeAlert) {
            // check % growth if no absolute threshold set & !enablePercentChangeAlert
            CountStat growthIncrease = calculateStats(latest, previous.getNumRows());
            if (growthIncrease == null) {
                continue;
            }
            if (Math.abs(growthIncrease.percentageIncrease) > PERCENTAGE_THRESHOLD) {
                abovePercentageList.add(growthIncrease);
            }
        }
    }
    return new Twin<>(aboveThresholdList, abovePercentageList);
}

Example 3 with Twin

use of io.datarouter.util.tuple.Twin in project datarouter by hotpads.

the class WebappInstanceAlertJob method makeContent.

private ContainerTag<?> makeContent(WebappInstance webappInstance, DatarouterDuration buildAge) {
    ZoneId zoneId = defaultDistributionListZoneId.get();
    var rows = List.of(new Twin<>("webapp", webappInstance.getKey().getWebappName()), new Twin<>("build date", ZonedDateFormatterTool.formatInstantWithZone(webappInstance.getBuildInstant(), zoneId)), new Twin<>("build age", buildAge.toString()), new Twin<>("startup date", ZonedDateFormatterTool.formatInstantWithZone(webappInstance.getStartupInstant(), zoneId)), new Twin<>("commitId", webappInstance.getCommitId()));
    return new J2HtmlEmailTable<Twin<String>>().withColumn(new J2HtmlEmailTableColumn<>(null, row -> makeDivBoldRight(row.getLeft()))).withColumn(new J2HtmlEmailTableColumn<>(null, row -> text(row.getRight()))).build(rows);
}

Example 4 with Twin

use of io.datarouter.util.tuple.Twin in project datarouter by hotpads.

the class ViewNodeDataHandler method countKeys.

@Handler
public <PK extends PrimaryKey<PK>, D extends Databean<PK, D>> Mav countKeys(OptionalInteger batchSize, OptionalInteger logBatchSize, OptionalInteger limit, OptionalBoolean useOffsetting, OptionalInteger stride) {
    showForm();
    if (!(node instanceof SortedStorageWriter<?, ?>)) {
        return pageFactory.message(request, "Cannot browse unsorted node");
    }
    PhysicalNode<?, ?, ?> physicalNode = NodeTool.extractSinglePhysicalNode(node);
    // TODO replace strings with more formal client detection
    boolean actualUseOffsetting = useOffsetting.orElse(physicalNode.getClientType().supportsOffsetSampling());
    @SuppressWarnings("unchecked") SortedStorageReader<PK, D> sortedNode = (SortedStorageReader<PK, D>) node;
    long count = 0;
    long startMs = System.currentTimeMillis() - 1;
    PK last = null;
    if (actualUseOffsetting) {
        var countingToolBuilder = new StrideScannerBuilder<>(sortedNode).withLog(true);
        stride.ifPresent(countingToolBuilder::withStride);
        batchSize.ifPresent(countingToolBuilder::withBatchSize);
        count = countingToolBuilder.build().findLast().map(sample -> sample.totalCount).orElse(0L);
    } else {
        Config config = new Config().setResponseBatchSize(batchSize.orElse(1000)).setScannerCaching(// disabled due to BigTable bug?
        false).setTimeout(Duration.ofMinutes(1)).anyDelay().setNumAttempts(1);
        limit.ifPresent(config::setLimit);
        int printBatchSize = logBatchSize.orElse(100_000);
        long batchStartMs = System.currentTimeMillis() - 1;
        for (PK pk : sortedNode.scanKeys(config).iterable()) {
            if (ComparableTool.lt(pk, last)) {
                // shouldn't happen, but seems to once in 10mm times
                logger.warn("{} was < {}", pk, last);
            }
            ++count;
            if (count % printBatchSize == 0) {
                long batchMs = System.currentTimeMillis() - batchStartMs;
                double batchAvgRps = printBatchSize * 1000 / Math.max(1, batchMs);
                logger.warn("{} {} {} @{}rps", NumberFormatter.addCommas(count), node.getName(), pk.toString(), NumberFormatter.addCommas(batchAvgRps));
                batchStartMs = System.currentTimeMillis();
            }
            last = pk;
        }
    }
    if (count < 1) {
        return pageFactory.message(request, "no rows found");
    }
    long endMs = System.currentTimeMillis();
    long durationMs = endMs - startMs;
    DatarouterDuration duration = new DatarouterDuration(durationMs, TimeUnit.MILLISECONDS);
    double avgRps = count * 1_000 / durationMs;
    String message = String.format("finished counting %s at %s %s @%srps totalDuration=%s", node.getName(), NumberFormatter.addCommas(count), last == null ? "?" : last.toString(), NumberFormatter.addCommas(avgRps), duration);
    logger.warn(message);
    List<Twin<String>> emailKvs = List.of(Twin.of("node", node.getName()), Twin.of("useOffsetting", actualUseOffsetting + ""), Twin.of("stride", stride.map(Object::toString).orElse("default")), Twin.of("totalCount", NumberFormatter.addCommas(count)), Twin.of("lastKey", last == null ? "?" : last.toString()), Twin.of("averageRps", NumberFormatter.addCommas(avgRps)), Twin.of("start", ZonedDateFormatterTool.formatLongMsWithZone(startMs, ZoneId.systemDefault())), Twin.of("end", ZonedDateFormatterTool.formatLongMsWithZone(endMs, ZoneId.systemDefault())), Twin.of("duration", duration + ""), Twin.of("triggeredBy", getSessionInfo().getRequiredSession().getUsername()));
    sendEmail(node.getName(), emailKvs);
    var dto = new DatarouterChangelogDtoBuilder("Inspect Node Data", node.getName(), "countKeys", getSessionInfo().getRequiredSession().getUsername()).build();
    changelogRecorder.record(dto);
    return pageFactory.message(request, message);
}

Example 5 with Twin

use of io.datarouter.util.tuple.Twin in project datarouter by hotpads.

the class SqsQueueRegistryService method getSqsQueuesForClient.

public Pair<List<Twin<String>>, List<String>> getSqsQueuesForClient(ClientId clientId) {
    Set<String> knownQueuesUrls = new HashSet<>();
    AmazonSQS sqs = sqsClientManager.getAmazonSqs(clientId);
    List<? extends BaseSqsNode<?, ?, ?>> sqsNodes = Scanner.of(nodes.getPhysicalNodesForClient(clientId.getName())).map(NodeTool::extractSinglePhysicalNode).map(physicalNode -> (BaseSqsNode<?, ?, ?>) physicalNode).list();
    List<Twin<String>> knownQueueUrlByName = Scanner.of(sqsNodes).map(BaseSqsNode::getQueueUrlAndName).map(Supplier::get).each(twin -> knownQueuesUrls.add(twin.getLeft())).list();
    List<String> unreferencedQueues = Scanner.of(sqsNodes).map(BaseSqsNode::buildNamespace).distinct().map(sqs::listQueues).concatIter(ListQueuesResult::getQueueUrls).exclude(knownQueuesUrls::contains).map(queueUrl -> StringTool.getStringAfterLastOccurrence("/", queueUrl)).include(queueName -> sqsQueueExists(sqs, queueName)).list();
    return new Pair<>(knownQueueUrlByName, unreferencedQueues);
}