Examples with CollectedUpdateMetrics org.wikidata.query.rdf.tool.rdf.CollectedUpdateMetrics used on opensource projects

Example 1 with CollectedUpdateMetrics

use of org.wikidata.query.rdf.tool.rdf.CollectedUpdateMetrics in project wikidata-query-rdf by wikimedia.

the class UpdaterMetricsRepository method updateChanges.

public void updateChanges(Supplier<CollectedUpdateMetrics> supplier, long changesCount) {
    CollectedUpdateMetrics updateMetrics = rdfRepositoryImportTime.time(supplier);
    updatesMeter.mark(changesCount);
    importedChanged.inc(changesCount);
    updateMetricsFrom(updateMetrics);
}

Example 2 with CollectedUpdateMetrics

use of org.wikidata.query.rdf.tool.rdf.CollectedUpdateMetrics in project wikidata-query-rdf by wikimedia.

the class UpdaterUnitTest method testUpdateLeftOffTime.

@Test
public void testUpdateLeftOffTime() {
    Instant leftOffInstant1 = Instant.ofEpochMilli(25);
    Instant leftOffInstant2 = Instant.ofEpochSecond(40);
    ImmutableList<Change> changes = ImmutableList.of(new Change("Q2", 1, Instant.ofEpochSecond(10), 2), new Change("Q3", 2, Instant.ofEpochMilli(20), 3));
    TestChange batch1 = new TestChange(changes, 20, leftOffInstant1, false);
    changes = ImmutableList.of(new Change("Q2", 1, Instant.ofEpochSecond(30), 4), new Change("Q3", 2, Instant.ofEpochMilli(40), 5));
    TestChange batch2 = new TestChange(changes, 20, leftOffInstant2, true);
    TestChangeSource source = new TestChangeSource(Arrays.asList(batch1, batch2));
    WikibaseRepository wbRepo = mock(WikibaseRepository.class);
    RdfRepository rdfRepo = mock(RdfRepository.class);
    CollectedUpdateMetrics mutationCountOnlyMetrics = CollectedUpdateMetrics.getMutationCountOnlyMetrics(0);
    when(rdfRepo.syncFromChanges(anyCollectionOf(Change.class), anyBoolean())).thenReturn(mutationCountOnlyMetrics);
    Munger munger = Munger.builder(UrisSchemeFactory.WIKIDATA).build();
    ExecutorService executorService = Executors.newFixedThreadPool(2, (r) -> new Thread(r, "Thread-" + this.getClass().getSimpleName()));
    MetricRegistry metricRegistry = new MetricRegistry();
    Updater<TestChange> updater = new Updater<>(source, wbRepo, rdfRepo, munger, executorService, true, 100, UrisSchemeFactory.WIKIDATA, false, metricRegistry);
    updater.run();
    verify(rdfRepo, times(2)).updateLeftOffTime(lestOffDateCaptor.capture());
    assertThat(lestOffDateCaptor.getAllValues()).containsExactly(leftOffInstant1.minusSeconds(1), leftOffInstant2.minusSeconds(1));
    assertThat(source.isBatchMarkedDone(batch1)).isTrue();
    assertThat(source.isBatchMarkedDone(batch2)).isTrue();
}

Example 3 with CollectedUpdateMetrics

use of org.wikidata.query.rdf.tool.rdf.CollectedUpdateMetrics in project wikidata-query-rdf by wikimedia.

the class UpdateMetricsResponseHandlerUnitTest method shouldNotThrowExceptionOnMalformedInput.

@Test
public void shouldNotThrowExceptionOnMalformedInput() throws IOException {
    String content = loadIncorrectResponseFromFile();
    setupResponse(content);
    CollectedUpdateMetrics collectedUpdateMetrics = responseHandler.parse(response);
    UpdateMetrics updateMetrics = collectedUpdateMetrics.getMetrics(CLEAR_OOD_SITE_LINKS);
    assertThat(collectedUpdateMetrics.getMutationCount()).isEqualTo(0);
    assertThat(collectedUpdateMetrics.getCommitTotalElapsed()).isEqualTo(0);
    assertThat(updateMetrics.getTotalElapsed()).isEqualTo(null);
    assertThat(updateMetrics.getElapsed()).isEqualTo(null);
    assertThat(updateMetrics.getConnFlush()).isEqualTo(null);
    assertThat(updateMetrics.getBatchResolve()).isEqualTo(null);
    assertThat(updateMetrics.getWhereClause()).isEqualTo(null);
    assertThat(updateMetrics.getDeleteClause()).isEqualTo(null);
    assertThat(updateMetrics.getInsertClause()).isEqualTo(null);
}

Example 4 with CollectedUpdateMetrics

use of org.wikidata.query.rdf.tool.rdf.CollectedUpdateMetrics in project wikidata-query-rdf by wikimedia.

the class UpdateMetricsResponseHandlerUnitTest method canParseUpdateMetrics.

@Test
public void canParseUpdateMetrics() throws IOException {
    String content = loadCorrectResponseFromFile();
    setupResponse(content);
    CollectedUpdateMetrics collectedUpdateMetrics = responseHandler.parse(response);
    for (Map.Entry<MultiSyncStep, List<Integer>> entry : expectedMetrics.entrySet()) {
        UpdateMetrics updateMetrics = collectedUpdateMetrics.getMetrics(entry.getKey());
        List<Integer> metrics = entry.getValue();
        assertThat(updateMetrics.getTotalElapsed()).isEqualTo(metrics.get(0));
        assertThat(updateMetrics.getElapsed()).isEqualTo(metrics.get(1));
        assertThat(updateMetrics.getConnFlush()).isEqualTo(metrics.get(2));
        assertThat(updateMetrics.getBatchResolve()).isEqualTo(metrics.get(3));
        assertThat(updateMetrics.getWhereClause()).isEqualTo(metrics.get(4));
        assertThat(updateMetrics.getDeleteClause()).isEqualTo(metrics.get(5));
        assertThat(updateMetrics.getInsertClause()).isEqualTo(metrics.get(6));
    }
    assertThat(collectedUpdateMetrics.getMutationCount()).isEqualTo(4);
    assertThat(collectedUpdateMetrics.getCommitTotalElapsed()).isEqualTo(29);
}

Example 5 with CollectedUpdateMetrics

use of org.wikidata.query.rdf.tool.rdf.CollectedUpdateMetrics in project wikidata-query-rdf by wikimedia.

the class UpdaterMetricsRepositoryUnitTest method testBlazegraphMetrics.

@Test
public void testBlazegraphMetrics() {
    CollectedUpdateMetrics collectedUpdateMetrics = new CollectedUpdateMetrics();
    collectedUpdateMetrics.setMutationCount(10);
    collectedUpdateMetrics.setCommitTotalElapsed(45);
    for (MultiSyncStep step : MultiSyncStep.values()) {
        collectedUpdateMetrics.merge(step, createSampleUpdateMetrics());
    }
    updaterMetricsRepository.updateChanges(() -> collectedUpdateMetrics, 10);
    for (MultiSyncStep step : MultiSyncStep.values()) {
        String base = step.getMetricBaseName();
        SortedMap<String, Counter> counters = metricRegistry.getCounters(MetricFilter.startsWith(base));
        assertThat(counters.size()).isEqualTo(MultiSyncStep.values().length);
        assertThat(counters.get(base + "elapsed").getCount()).isEqualTo(5);
        assertThat(counters.get(base + "batch-resolve").getCount()).isEqualTo(0);
        assertThat(counters.get(base + "delete-clause").getCount()).isEqualTo(0);
        assertThat(counters.get(base + "insert-clause").getCount()).isEqualTo(1);
        assertThat(counters.get(base + "total-elapsed").getCount()).isEqualTo(4);
        assertThat(counters.get(base + "conn-flush").getCount()).isEqualTo(1);
        assertThat(counters.get(base + "where-clause").getCount()).isEqualTo(3);
    }
    assertThat(metricRegistry.counter("rdf-repository-imported-triples").getCount()).isEqualTo(10);
    assertThat(metricRegistry.counter("rdf-repository-imported-changes").getCount()).isEqualTo(10);
    assertThat(metricRegistry.counter("blazegraph-commit-total-elapsed").getCount()).isEqualTo(45);
}