Examples with MultiSyncStep org.wikidata.query.rdf.tool.rdf.MultiSyncStep used on opensource projects

Example 1 with MultiSyncStep

use of org.wikidata.query.rdf.tool.rdf.MultiSyncStep in project wikidata-query-rdf by wikimedia.

the class UpdateMetricsResponseHandlerUnitTest method canParseUpdateMetrics.

@Test
public void canParseUpdateMetrics() throws IOException {
    String content = loadCorrectResponseFromFile();
    setupResponse(content);
    CollectedUpdateMetrics collectedUpdateMetrics = responseHandler.parse(response);
    for (Map.Entry<MultiSyncStep, List<Integer>> entry : expectedMetrics.entrySet()) {
        UpdateMetrics updateMetrics = collectedUpdateMetrics.getMetrics(entry.getKey());
        List<Integer> metrics = entry.getValue();
        assertThat(updateMetrics.getTotalElapsed()).isEqualTo(metrics.get(0));
        assertThat(updateMetrics.getElapsed()).isEqualTo(metrics.get(1));
        assertThat(updateMetrics.getConnFlush()).isEqualTo(metrics.get(2));
        assertThat(updateMetrics.getBatchResolve()).isEqualTo(metrics.get(3));
        assertThat(updateMetrics.getWhereClause()).isEqualTo(metrics.get(4));
        assertThat(updateMetrics.getDeleteClause()).isEqualTo(metrics.get(5));
        assertThat(updateMetrics.getInsertClause()).isEqualTo(metrics.get(6));
    }
    assertThat(collectedUpdateMetrics.getMutationCount()).isEqualTo(4);
    assertThat(collectedUpdateMetrics.getCommitTotalElapsed()).isEqualTo(29);
}

Example 2 with MultiSyncStep

use of org.wikidata.query.rdf.tool.rdf.MultiSyncStep in project wikidata-query-rdf by wikimedia.

the class UpdaterMetricsRepositoryUnitTest method testBlazegraphMetrics.

@Test
public void testBlazegraphMetrics() {
    CollectedUpdateMetrics collectedUpdateMetrics = new CollectedUpdateMetrics();
    collectedUpdateMetrics.setMutationCount(10);
    collectedUpdateMetrics.setCommitTotalElapsed(45);
    for (MultiSyncStep step : MultiSyncStep.values()) {
        collectedUpdateMetrics.merge(step, createSampleUpdateMetrics());
    }
    updaterMetricsRepository.updateChanges(() -> collectedUpdateMetrics, 10);
    for (MultiSyncStep step : MultiSyncStep.values()) {
        String base = step.getMetricBaseName();
        SortedMap<String, Counter> counters = metricRegistry.getCounters(MetricFilter.startsWith(base));
        assertThat(counters.size()).isEqualTo(MultiSyncStep.values().length);
        assertThat(counters.get(base + "elapsed").getCount()).isEqualTo(5);
        assertThat(counters.get(base + "batch-resolve").getCount()).isEqualTo(0);
        assertThat(counters.get(base + "delete-clause").getCount()).isEqualTo(0);
        assertThat(counters.get(base + "insert-clause").getCount()).isEqualTo(1);
        assertThat(counters.get(base + "total-elapsed").getCount()).isEqualTo(4);
        assertThat(counters.get(base + "conn-flush").getCount()).isEqualTo(1);
        assertThat(counters.get(base + "where-clause").getCount()).isEqualTo(3);
    }
    assertThat(metricRegistry.counter("rdf-repository-imported-triples").getCount()).isEqualTo(10);
    assertThat(metricRegistry.counter("rdf-repository-imported-changes").getCount()).isEqualTo(10);
    assertThat(metricRegistry.counter("blazegraph-commit-total-elapsed").getCount()).isEqualTo(45);
}

Example 3 with MultiSyncStep

use of org.wikidata.query.rdf.tool.rdf.MultiSyncStep in project wikidata-query-rdf by wikimedia.

the class UpdaterMetricsRepository method updateMetricsFrom.

private void updateMetricsFrom(CollectedUpdateMetrics updateMetrics) {
    importedTriples.inc(updateMetrics.getMutationCount());
    blazeGraphCommitTotalElapsed.inc(updateMetrics.getCommitTotalElapsed());
    for (MultiSyncStep step : updateMetrics.definedMetrics()) {
        blazeGraphMetrics.get(step).updateMetrics(updateMetrics.getMetrics(step));
    }
}

Example 4 with MultiSyncStep

use of org.wikidata.query.rdf.tool.rdf.MultiSyncStep in project wikidata-query-rdf by wikimedia.

the class UpdateMetricsResponseHandler method extractMetrics.

private CollectedUpdateMetrics extractMetrics(String[] lines) throws IOException {
    CollectedUpdateMetrics collectedUpdateMetrics = new CollectedUpdateMetrics();
    Iterator<String> lineIterator = getPerQueryResponses(lines);
    for (MultiSyncStep step : requiredSteps) {
        if (lineIterator.hasNext()) {
            collectedUpdateMetrics.merge(step, getUpdateMetrics(lineIterator.next()));
        } else {
            throw new IOException("Response didn't match the query!");
        }
    }
    Map<String, String> summaryMetrics = extractSummaryMetrics(lines);
    Integer commitTotalElapsed = asInt(summaryMetrics.get(COMMIT_TOTAL_ELAPSED));
    if (commitTotalElapsed != null)
        collectedUpdateMetrics.setCommitTotalElapsed(commitTotalElapsed);
    Integer mutationCount = asInt(summaryMetrics.get(MUTATION_COUNT));
    if (mutationCount == null)
        throw new IOException("Couldn't find the mutation count!");
    collectedUpdateMetrics.setMutationCount(mutationCount);
    return collectedUpdateMetrics;
}