Examples with RDFPatchResult org.wikidata.query.rdf.tool.rdf.RDFPatchResult used on opensource projects

Example 1 with RDFPatchResult

use of org.wikidata.query.rdf.tool.rdf.RDFPatchResult in project wikidata-query-rdf by wikimedia.

the class StreamingUpdaterConsumer method run.

public void run() {
    try (StreamConsumer consumer = this.consumer;
        RdfRepositoryUpdater repository = this.repository) {
        // as unavailable preventing offsets to be committed.
        while (!stop) {
            StreamConsumer.Batch b = pollTimeCnt.time(() -> consumer.poll(TIMEOUT));
            if (b == null) {
                continue;
            }
            RDFPatchResult result = rdfStoreTimeCnt.time(() -> repository.applyPatch(b.getPatch(), b.getAverageEventTime()));
            updateCounters(result);
            if (passInconsistencyThreshold(result, inconsistencyWarningThreshold)) {
                logger.warn("Applied batch with too many inconsistencies. {} for {}.", result, b);
            }
            consumer.acknowledge();
        }
    }
}

Example 2 with RDFPatchResult

use of org.wikidata.query.rdf.tool.rdf.RDFPatchResult in project wikidata-query-rdf by wikimedia.

the class StreamingUpdaterConsumerUnitTest method test.

@Test
public void test() throws InterruptedException {
    List<String> entityIdsToDelete = new ArrayList<String>();
    Map<String, Collection<Statement>> reconciliations = new HashMap<>();
    ConsumerPatch patch = new ConsumerPatch(statements(), statements(), statements(), statements(), entityIdsToDelete, reconciliations);
    Instant avgEventTime = Instant.EPOCH.plus(4, ChronoUnit.MINUTES);
    RDFPatchResult rdfPatchResult = new RDFPatchResult(2, 1, 2, 1, 1, 1);
    LongAdder patchApplied = new LongAdder();
    CountDownLatch countdown = new CountDownLatch(5);
    Answer<StreamConsumer.Batch> batchSupplier = (i) -> new StreamConsumer.Batch(patch, avgEventTime, "1", Instant.now(), "2", Instant.now());
    when(consumer.poll(any())).thenAnswer(batchSupplier);
    when(rdfRepositoryUpdater.applyPatch(any(), any())).thenAnswer((Answer<RDFPatchResult>) i -> {
        countdown.countDown();
        patchApplied.increment();
        return rdfPatchResult;
    });
    MetricRegistry registry = new MetricRegistry();
    StreamingUpdaterConsumer updater = new StreamingUpdaterConsumer(consumer, rdfRepositoryUpdater, registry, 1F);
    Thread t = new Thread(updater);
    t.start();
    // Wait for five patches to be applied and stop the updater
    countdown.await();
    updater.close();
    t.join();
    // Make sure that we called the methods the right number of times
    // This updater does not much other than bridging a consumer and a repository
    verify(consumer, times(patchApplied.intValue())).poll(any());
    verify(consumer, times(patchApplied.intValue())).acknowledge();
    verify(consumer, times(1)).close();
    verify(rdfRepositoryUpdater, times(patchApplied.intValue())).applyPatch(same(patch), same(avgEventTime));
    verify(rdfRepositoryUpdater, times(1)).close();
    assertThat(registry.counter("mutations").getCount()).isEqualTo(patchApplied.intValue());
    assertThat(registry.counter("delete-mutations").getCount()).isEqualTo(patchApplied.intValue());
    assertThat(registry.counter("reconciliation-mutations").getCount()).isEqualTo(patchApplied.intValue());
    assertThat(registry.counter("divergences").getCount()).isEqualTo(patchApplied.intValue());
    assertThat(registry.counter("shared-element-mutations").getCount()).isEqualTo(patchApplied.intValue());
    assertThat(registry.counter("shared-element-redundant-mutations").getCount()).isEqualTo(patchApplied.intValue());
}

Example 3 with RDFPatchResult

use of org.wikidata.query.rdf.tool.rdf.RDFPatchResult in project wikidata-query-rdf by wikimedia.

the class StreamingUpdaterConsumerUnitTest method testInconsistenciesThreshold.

@Test
public void testInconsistenciesThreshold() {
    RDFPatchResult res = new RDFPatchResult(100, 98, 0, 0, 0, 0);
    assertThat(passInconsistencyThreshold(res, 0.02F)).isFalse();
    assertThat(passInconsistencyThreshold(res, 0.01F)).isTrue();
    res = new RDFPatchResult(0, 99, 0, 0, 0, 0);
    assertThat(passInconsistencyThreshold(res, 0.01F)).isFalse();
}