Search in sources :

Example 6 with ChangeEvent

use of org.wikidata.query.rdf.tool.change.events.ChangeEvent in project wikidata-query-rdf by wikimedia.

the class KafkaPollerUnitTest method multiPolls.

@Test
public void multiPolls() throws RetryableException {
    KafkaPoller poller = makePoller();
    ConsumerRecords<String, ChangeEvent> rs1 = makeRecords(makeRecord(makeRCEvent(Duration.ofMillis(20), 1, 5, "Q123"), "topictest", Duration.ofMillis(20)), makeRecord(makeRCEvent(Duration.ofMillis(30), 2, 2, "Q666", 1, DOMAIN), "othertopic", Duration.ofMillis(21)), makeRecord(makeRCEvent(Duration.ofMillis(25), 3, 10, "Q6666", 0, "acme.wrong"), "topictest", Duration.ofMillis(20)));
    ConsumerRecords<String, ChangeEvent> rs2 = makeRecords(makeRecord(makeRCEvent(Duration.ofMillis(30), 2, 2, "Q666", 1, DOMAIN), "othertopic", Duration.ofMillis(21)), makeRecord(makeRCEvent(Duration.ofMillis(25), 3, 10, "Q6666", 0, "acme.wrong"), "topictest", Duration.ofMillis(20)));
    ConsumerRecords<String, ChangeEvent> rs3 = makeRecords(makeRecord(makeRCEvent(Duration.ofMillis(30), 4, 2, "Q234"), "othertopic", Duration.ofMillis(21)), makeRecord(makeRCEvent(Duration.ofMillis(25), 3, 10, "Q6666", 0, "acme.wrong"), "topictest", Duration.ofMillis(20)), makeRecord(makeRCEvent(Duration.ofMillis(30), 1, 10, "Q123"), "othertopic", Duration.ofMillis(31)), makeRecord(makeRCEvent(Duration.ofMillis(30), 5, 21, "Q245"), "topictest", Duration.ofMillis(40)));
    when(consumer.poll(anyLong())).thenReturn(rs1, rs2, rs3, EMPTY_CHANGES);
    Batch batch = poller.firstBatch();
    // second batch did not have good messages, so the poller should return
    // before third batch
    assertThat(batch.changes()).hasSize(1).anyMatch(titleRevision("Q123", 5));
    batch = poller.nextBatch(batch);
    assertThat(batch.changes()).hasSize(3).anyMatch(titleRevision("Q123", 10)).anyMatch(title("Q234")).anyMatch(title("Q245"));
}
Also used : ChangeEvent(org.wikidata.query.rdf.tool.change.events.ChangeEvent) Batch(org.wikidata.query.rdf.tool.change.KafkaPoller.Batch) Test(org.junit.Test)

Example 7 with ChangeEvent

use of org.wikidata.query.rdf.tool.change.events.ChangeEvent in project wikidata-query-rdf by wikimedia.

the class KafkaPollerUnitTest method newChange.

private ChangeEvent newChange(String enityId) {
    ChangeEvent e = mock(ChangeEvent.class);
    when(e.domain()).thenReturn(uris.getHost());
    when(e.title()).thenReturn(enityId);
    when(e.revision()).thenReturn(Change.NO_REVISION);
    when(e.namespace()).thenReturn(ENTITY_NS);
    return e;
}
Also used : ChangeEvent(org.wikidata.query.rdf.tool.change.events.ChangeEvent)

Example 8 with ChangeEvent

use of org.wikidata.query.rdf.tool.change.events.ChangeEvent in project wikidata-query-rdf by wikimedia.

the class KafkaPollerUnitTest method advanceTimestamp.

@Test
public void advanceTimestamp() throws RetryableException {
    ConsumerRecords<String, ChangeEvent> rs = makeRecords(makeRecord(makeRCEvent(Duration.ofMillis(20), 1, 1, "Q123"), "mediawiki.revision-create", Duration.ofMillis(120000)), makeRecord(makeRCEvent(Duration.ofMillis(30), 2, 2, "Q234"), "mediawiki.revision-create", Duration.ofMillis(122000)), makeRecord(makeDeleteEvent(Duration.ofMillis(21), 1, "Q123"), "othertopic", Duration.ofMillis(121000)), makeRecord(makeDeleteEvent(Duration.ofMillis(22), 2, "Q234"), "othertopic", Duration.ofMillis(122000)), makeRecord(makeDeleteEvent(Duration.ofMillis(31), 1, "Q123"), "othertopic", Duration.ofMillis(123000)));
    Batch batch = getBatchFromRecords(rs);
    // Advancement is minimum over maximal times of the topics
    assertThat(batch.advanced()).isEqualTo(122000L);
    assertThat(batch.leftOffDate()).isEqualTo(START_TIME.plusMillis(122000L));
}
Also used : ChangeEvent(org.wikidata.query.rdf.tool.change.events.ChangeEvent) Batch(org.wikidata.query.rdf.tool.change.KafkaPoller.Batch) Test(org.junit.Test)

Example 9 with ChangeEvent

use of org.wikidata.query.rdf.tool.change.events.ChangeEvent in project wikidata-query-rdf by wikimedia.

the class KafkaPollerEventConsumptionUnitTest method initPoller.

private void initPoller(String... clusterNames) {
    Uris uris = Uris.withWikidataDefaults("https://acme.test");
    URI root = null;
    try {
        root = uris.builder().build();
    } catch (URISyntaxException e) {
        fail("failed to build UriScheme", e);
    }
    KafkaOffsetsRepository kafkaOffsetsRepository = new RdfKafkaOffsetsRepository(root, null);
    Map<String, Class<? extends ChangeEvent>> topicsToClass = KafkaPoller.clusterNamesAwareTopics(Arrays.asList(clusterNames));
    deserializer = new JsonDeserializer<>(topicsToClass);
    poller = new KafkaPoller(consumer, uris, Instant.now(), 100, topicsToClass.keySet(), kafkaOffsetsRepository, true, new MetricRegistry());
}
Also used : ChangeEvent(org.wikidata.query.rdf.tool.change.events.ChangeEvent) MetricRegistry(com.codahale.metrics.MetricRegistry) URISyntaxException(java.net.URISyntaxException) URI(java.net.URI) Uris(org.wikidata.query.rdf.tool.wikibase.WikibaseRepository.Uris)

Example 10 with ChangeEvent

use of org.wikidata.query.rdf.tool.change.events.ChangeEvent in project wikidata-query-rdf by wikimedia.

the class KafkaPollerUnitTest method changesOrder.

@Test
public void changesOrder() throws RetryableException {
    ConsumerRecords<String, ChangeEvent> rs = makeRecords(makeRecord(makeRCEvent(Duration.ofMillis(20), 1, 5, "Q123"), "topictest", Duration.ofMillis(20)), makeRecord(makeRCEvent(Duration.ofMillis(30), 1, 2, "Q123"), "othertopic", Duration.ofMillis(21)), makeRecord(makeRCEvent(Duration.ofMillis(25), 1, 10, "Q123"), "othertopic", Duration.ofMillis(15)), makeRecord(makeRCEvent(Duration.ofMillis(35), 1, 7, "Q123"), "topictest", Duration.ofMillis(25)));
    Batch batch = getBatchFromRecords(rs);
    // There should be only one change, and it should have max revision
    assertThat(batch.changes()).hasSize(1).anyMatch(revision(10));
}
Also used : ChangeEvent(org.wikidata.query.rdf.tool.change.events.ChangeEvent) Batch(org.wikidata.query.rdf.tool.change.KafkaPoller.Batch) Test(org.junit.Test)

Aggregations

ChangeEvent (org.wikidata.query.rdf.tool.change.events.ChangeEvent)12 Test (org.junit.Test)9 Batch (org.wikidata.query.rdf.tool.change.KafkaPoller.Batch)9 MetricRegistry (com.codahale.metrics.MetricRegistry)2 HashMap (java.util.HashMap)2 OffsetAndMetadata (org.apache.kafka.clients.consumer.OffsetAndMetadata)2 TopicPartition (org.apache.kafka.common.TopicPartition)2 Context (com.codahale.metrics.Timer.Context)1 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 URI (java.net.URI)1 URISyntaxException (java.net.URISyntaxException)1 Instant (java.time.Instant)1 ArrayList (java.util.ArrayList)1 Collections.singletonList (java.util.Collections.singletonList)1 LinkedHashMap (java.util.LinkedHashMap)1 List (java.util.List)1 Map (java.util.Map)1 Collectors.toList (java.util.stream.Collectors.toList)1 ConsumerRecord (org.apache.kafka.clients.consumer.ConsumerRecord)1