use of org.wikidata.wdtk.datamodel.interfaces.ItemDocument in project OpenRefine by OpenRefine.
the class EditBatchProcessorTest method testMultipleBatches.
@Test
public void testMultipleBatches() throws MediaWikiApiErrorException, InterruptedException, IOException {
// Prepare test data
MonolingualTextValue description = Datamodel.makeMonolingualTextValue("village in Nepal", "en");
List<String> ids = new ArrayList<>();
for (int i = 124; i < 190; i++) {
ids.add("Q" + String.valueOf(i));
}
List<ItemIdValue> qids = ids.stream().map(e -> Datamodel.makeWikidataItemIdValue(e)).collect(Collectors.toList());
List<TermedStatementEntityEdit> batch = qids.stream().map(qid -> new TermedStatementEntityEditBuilder(qid).addDescription(description, true).build()).collect(Collectors.toList());
int batchSize = 50;
List<ItemDocument> fullBatch = qids.stream().map(qid -> ItemDocumentBuilder.forItemId(qid).withStatement(TestingData.generateStatement(qid, TestingData.existingId)).build()).collect(Collectors.toList());
List<ItemDocument> firstBatch = fullBatch.subList(0, batchSize);
List<ItemDocument> secondBatch = fullBatch.subList(batchSize, fullBatch.size());
when(fetcher.getEntityDocuments(toQids(firstBatch))).thenReturn(toMap(firstBatch));
when(fetcher.getEntityDocuments(toQids(secondBatch))).thenReturn(toMap(secondBatch));
// Run edits
EditBatchProcessor processor = new EditBatchProcessor(fetcher, editor, batch, library, summary, maxlag, tags, batchSize);
assertEquals(0, processor.progress());
for (int i = 124; i < 190; i++) {
assertEquals(processor.remainingEdits(), 190 - i);
processor.performEdit();
}
assertEquals(0, processor.remainingEdits());
assertEquals(100, processor.progress());
// Check result
assertEquals(new NewEntityLibrary(), library);
verify(fetcher, times(1)).getEntityDocuments(toQids(firstBatch));
verify(fetcher, times(1)).getEntityDocuments(toQids(secondBatch));
for (ItemDocument doc : fullBatch) {
verify(editor, times(1)).editEntityDocument(Datamodel.makeItemUpdate(doc.getEntityId(), doc.getRevisionId(), Datamodel.makeTermUpdate(Collections.emptyList(), Collections.emptyList()), Datamodel.makeTermUpdate(Collections.singletonList(description), Collections.emptyList()), Collections.emptyMap(), Datamodel.makeStatementUpdate(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()), Collections.emptyList(), Collections.emptyList()), false, summary, tags);
}
}
use of org.wikidata.wdtk.datamodel.interfaces.ItemDocument in project OpenRefine by OpenRefine.
the class EditBatchProcessorTest method testNewItem.
@Test
public void testNewItem() throws InterruptedException, MediaWikiApiErrorException, IOException {
List<TermedStatementEntityEdit> batch = new ArrayList<>();
batch.add(new TermedStatementEntityEditBuilder(TestingData.existingId).addAlias(Datamodel.makeMonolingualTextValue("my new alias", "en")).addStatement(TestingData.generateStatementAddition(TestingData.existingId, TestingData.newIdA)).build());
MonolingualTextValue label = Datamodel.makeMonolingualTextValue("better label", "en");
batch.add(new TermedStatementEntityEditBuilder(TestingData.newIdA).addAlias(label).build());
// Plan expected edits
ItemDocument existingItem = ItemDocumentBuilder.forItemId(TestingData.existingId).withLabel(Datamodel.makeMonolingualTextValue("pomme", "fr")).withDescription(Datamodel.makeMonolingualTextValue("fruit délicieux", "fr")).build();
when(fetcher.getEntityDocuments(Collections.singletonList(TestingData.existingId.getId()))).thenReturn(Collections.singletonMap(TestingData.existingId.getId(), existingItem));
ItemDocument expectedNewItem = ItemDocumentBuilder.forItemId(TestingData.newIdA).withLabel(label).build();
ItemDocument createdNewItem = ItemDocumentBuilder.forItemId(Datamodel.makeWikidataItemIdValue("Q1234")).withLabel(label).withRevisionId(37828L).build();
when(editor.createItemDocument(expectedNewItem, summary, tags)).thenReturn(createdNewItem);
EditBatchProcessor processor = new EditBatchProcessor(fetcher, editor, batch, library, summary, maxlag, tags, 50);
assertEquals(2, processor.remainingEdits());
assertEquals(0, processor.progress());
processor.performEdit();
assertEquals(1, processor.remainingEdits());
assertEquals(50, processor.progress());
processor.performEdit();
assertEquals(0, processor.remainingEdits());
assertEquals(100, processor.progress());
// does not do anything
processor.performEdit();
assertEquals(0, processor.remainingEdits());
assertEquals(100, processor.progress());
NewEntityLibrary expectedLibrary = new NewEntityLibrary();
expectedLibrary.setId(1234L, "Q1234");
assertEquals(expectedLibrary, library);
}
use of org.wikidata.wdtk.datamodel.interfaces.ItemDocument in project OpenRefine by OpenRefine.
the class TermedStatementEntityEditTest method testToEntityUpdate.
@Test
public void testToEntityUpdate() {
TermedStatementEntityEdit edit = new TermedStatementEntityEditBuilder(existingSubject).addAlias(Datamodel.makeMonolingualTextValue("alias", "en")).addStatement(statementUpdate1).build();
ItemDocument itemDocument = ItemDocumentBuilder.forItemId(existingSubject).withStatement(statement2).withLabel(Datamodel.makeMonolingualTextValue("label", "en")).build();
ItemUpdate update = (ItemUpdate) edit.toEntityUpdate(itemDocument);
assertEquals(update.getEntityId(), existingSubject);
assertEquals(update.getAliases(), Collections.singletonMap("en", Datamodel.makeAliasUpdate(Collections.singletonList(Datamodel.makeMonolingualTextValue("alias", "en")), Collections.emptyList())));
assertTrue(update.getDescriptions().isEmpty());
assertTrue(update.getLabels().isEmpty());
assertEquals(update.getStatements(), Datamodel.makeStatementUpdate(Collections.singletonList(statement1), Collections.emptyList(), Collections.emptySet()));
}
use of org.wikidata.wdtk.datamodel.interfaces.ItemDocument in project OpenRefine by OpenRefine.
the class TermedStatementEntityEditTest method testToNewEntity.
@Test
public void testToNewEntity() {
TermedStatementEntityEdit edit = new TermedStatementEntityEditBuilder(newSubject).addLabel(Datamodel.makeMonolingualTextValue("fr", "bonjour"), true).addDescription(Datamodel.makeMonolingualTextValue("de", "Redewendung"), true).build();
ItemDocument itemDocument = (ItemDocument) edit.toNewEntity();
ItemDocument expected = ItemDocumentBuilder.forItemId(newSubject).withLabel(Datamodel.makeMonolingualTextValue("fr", "bonjour")).withDescription(Datamodel.makeMonolingualTextValue("de", "Redewendung")).build();
assertEquals(itemDocument, expected);
}
use of org.wikidata.wdtk.datamodel.interfaces.ItemDocument in project OpenRefine by OpenRefine.
the class TermedStatementEntityEdit method toEntityUpdate.
/**
* In case the subject id is not new, returns the corresponding update given
* the current state of the entity.
*/
public EntityUpdate toEntityUpdate(EntityDocument entityDocument) {
Validate.isFalse(isNew(), "Cannot create a corresponding entity update for a creation of a new entity.");
if (id instanceof ItemIdValue) {
ItemDocument itemDocument = (ItemDocument) entityDocument;
// Labels
List<MonolingualTextValue> labels = getLabels().stream().collect(Collectors.toList());
labels.addAll(getLabelsIfNew().stream().filter(label -> !itemDocument.getLabels().containsKey(label.getLanguageCode())).collect(Collectors.toList()));
TermUpdate labelUpdate = Datamodel.makeTermUpdate(labels, Collections.emptyList());
// Descriptions
List<MonolingualTextValue> descriptions = getDescriptions().stream().collect(Collectors.toList());
descriptions.addAll(getDescriptionsIfNew().stream().filter(desc -> !itemDocument.getDescriptions().containsKey(desc.getLanguageCode())).collect(Collectors.toList()));
TermUpdate descriptionUpdate = Datamodel.makeTermUpdate(descriptions, Collections.emptyList());
// Aliases
Set<MonolingualTextValue> aliases = getAliases();
Map<String, List<MonolingualTextValue>> aliasesMap = aliases.stream().collect(Collectors.groupingBy(MonolingualTextValue::getLanguageCode));
Map<String, AliasUpdate> aliasMap = aliasesMap.entrySet().stream().collect(Collectors.toMap(Entry::getKey, e -> Datamodel.makeAliasUpdate(e.getValue(), Collections.emptyList())));
// Statements
StatementUpdate statementUpdate = toStatementUpdate(itemDocument);
return Datamodel.makeItemUpdate((ItemIdValue) getEntityId(), entityDocument.getRevisionId(), labelUpdate, descriptionUpdate, aliasMap, statementUpdate, Collections.emptyList(), Collections.emptyList());
} else if (id instanceof MediaInfoIdValue) {
MediaInfoDocument mediaInfoDocument = (MediaInfoDocument) entityDocument;
// Labels (captions)
List<MonolingualTextValue> labels = getLabels().stream().collect(Collectors.toList());
labels.addAll(getLabelsIfNew().stream().filter(label -> !mediaInfoDocument.getLabels().containsKey(label.getLanguageCode())).collect(Collectors.toList()));
TermUpdate labelUpdate = Datamodel.makeTermUpdate(labels, Collections.emptyList());
// Statements
StatementUpdate statementUpdate = toStatementUpdate(mediaInfoDocument);
return Datamodel.makeMediaInfoUpdate((MediaInfoIdValue) id, entityDocument.getRevisionId(), labelUpdate, statementUpdate);
} else {
throw new NotImplementedException("Editing entities of type " + id.getEntityType() + " is not supported yet.");
}
}
Aggregations