use of org.wikidata.query.rdf.tool.change.events.ChangeEvent in project wikidata-query-rdf by wikimedia.
the class KafkaPollerUnitTest method multiPolls.
@Test
public void multiPolls() throws RetryableException {
KafkaPoller poller = makePoller();
ConsumerRecords<String, ChangeEvent> rs1 = makeRecords(makeRecord(makeRCEvent(Duration.ofMillis(20), 1, 5, "Q123"), "topictest", Duration.ofMillis(20)), makeRecord(makeRCEvent(Duration.ofMillis(30), 2, 2, "Q666", 1, DOMAIN), "othertopic", Duration.ofMillis(21)), makeRecord(makeRCEvent(Duration.ofMillis(25), 3, 10, "Q6666", 0, "acme.wrong"), "topictest", Duration.ofMillis(20)));
ConsumerRecords<String, ChangeEvent> rs2 = makeRecords(makeRecord(makeRCEvent(Duration.ofMillis(30), 2, 2, "Q666", 1, DOMAIN), "othertopic", Duration.ofMillis(21)), makeRecord(makeRCEvent(Duration.ofMillis(25), 3, 10, "Q6666", 0, "acme.wrong"), "topictest", Duration.ofMillis(20)));
ConsumerRecords<String, ChangeEvent> rs3 = makeRecords(makeRecord(makeRCEvent(Duration.ofMillis(30), 4, 2, "Q234"), "othertopic", Duration.ofMillis(21)), makeRecord(makeRCEvent(Duration.ofMillis(25), 3, 10, "Q6666", 0, "acme.wrong"), "topictest", Duration.ofMillis(20)), makeRecord(makeRCEvent(Duration.ofMillis(30), 1, 10, "Q123"), "othertopic", Duration.ofMillis(31)), makeRecord(makeRCEvent(Duration.ofMillis(30), 5, 21, "Q245"), "topictest", Duration.ofMillis(40)));
when(consumer.poll(anyLong())).thenReturn(rs1, rs2, rs3, EMPTY_CHANGES);
Batch batch = poller.firstBatch();
// second batch did not have good messages, so the poller should return
// before third batch
assertThat(batch.changes()).hasSize(1).anyMatch(titleRevision("Q123", 5));
batch = poller.nextBatch(batch);
assertThat(batch.changes()).hasSize(3).anyMatch(titleRevision("Q123", 10)).anyMatch(title("Q234")).anyMatch(title("Q245"));
}
use of org.wikidata.query.rdf.tool.change.events.ChangeEvent in project wikidata-query-rdf by wikimedia.
the class KafkaPollerUnitTest method newChange.
private ChangeEvent newChange(String enityId) {
ChangeEvent e = mock(ChangeEvent.class);
when(e.domain()).thenReturn(uris.getHost());
when(e.title()).thenReturn(enityId);
when(e.revision()).thenReturn(Change.NO_REVISION);
when(e.namespace()).thenReturn(ENTITY_NS);
return e;
}
use of org.wikidata.query.rdf.tool.change.events.ChangeEvent in project wikidata-query-rdf by wikimedia.
the class KafkaPollerUnitTest method advanceTimestamp.
@Test
public void advanceTimestamp() throws RetryableException {
ConsumerRecords<String, ChangeEvent> rs = makeRecords(makeRecord(makeRCEvent(Duration.ofMillis(20), 1, 1, "Q123"), "mediawiki.revision-create", Duration.ofMillis(120000)), makeRecord(makeRCEvent(Duration.ofMillis(30), 2, 2, "Q234"), "mediawiki.revision-create", Duration.ofMillis(122000)), makeRecord(makeDeleteEvent(Duration.ofMillis(21), 1, "Q123"), "othertopic", Duration.ofMillis(121000)), makeRecord(makeDeleteEvent(Duration.ofMillis(22), 2, "Q234"), "othertopic", Duration.ofMillis(122000)), makeRecord(makeDeleteEvent(Duration.ofMillis(31), 1, "Q123"), "othertopic", Duration.ofMillis(123000)));
Batch batch = getBatchFromRecords(rs);
// Advancement is minimum over maximal times of the topics
assertThat(batch.advanced()).isEqualTo(122000L);
assertThat(batch.leftOffDate()).isEqualTo(START_TIME.plusMillis(122000L));
}
use of org.wikidata.query.rdf.tool.change.events.ChangeEvent in project wikidata-query-rdf by wikimedia.
the class KafkaPollerEventConsumptionUnitTest method initPoller.
private void initPoller(String... clusterNames) {
Uris uris = Uris.withWikidataDefaults("https://acme.test");
URI root = null;
try {
root = uris.builder().build();
} catch (URISyntaxException e) {
fail("failed to build UriScheme", e);
}
KafkaOffsetsRepository kafkaOffsetsRepository = new RdfKafkaOffsetsRepository(root, null);
Map<String, Class<? extends ChangeEvent>> topicsToClass = KafkaPoller.clusterNamesAwareTopics(Arrays.asList(clusterNames));
deserializer = new JsonDeserializer<>(topicsToClass);
poller = new KafkaPoller(consumer, uris, Instant.now(), 100, topicsToClass.keySet(), kafkaOffsetsRepository, true, new MetricRegistry());
}
use of org.wikidata.query.rdf.tool.change.events.ChangeEvent in project wikidata-query-rdf by wikimedia.
the class KafkaPollerUnitTest method changesOrder.
@Test
public void changesOrder() throws RetryableException {
ConsumerRecords<String, ChangeEvent> rs = makeRecords(makeRecord(makeRCEvent(Duration.ofMillis(20), 1, 5, "Q123"), "topictest", Duration.ofMillis(20)), makeRecord(makeRCEvent(Duration.ofMillis(30), 1, 2, "Q123"), "othertopic", Duration.ofMillis(21)), makeRecord(makeRCEvent(Duration.ofMillis(25), 1, 10, "Q123"), "othertopic", Duration.ofMillis(15)), makeRecord(makeRCEvent(Duration.ofMillis(35), 1, 7, "Q123"), "topictest", Duration.ofMillis(25)));
Batch batch = getBatchFromRecords(rs);
// There should be only one change, and it should have max revision
assertThat(batch.changes()).hasSize(1).anyMatch(revision(10));
}
Aggregations