Search in sources :

Example 1 with ItemDocument

use of org.wikidata.wdtk.datamodel.interfaces.ItemDocument in project OpenRefine by OpenRefine.

the class EditBatchProcessorTest method testMultipleBatches.

@Test
public void testMultipleBatches() throws MediaWikiApiErrorException, InterruptedException, IOException {
    // Prepare test data
    MonolingualTextValue description = Datamodel.makeMonolingualTextValue("village in Nepal", "en");
    List<String> ids = new ArrayList<>();
    for (int i = 124; i < 190; i++) {
        ids.add("Q" + String.valueOf(i));
    }
    List<ItemIdValue> qids = ids.stream().map(e -> Datamodel.makeWikidataItemIdValue(e)).collect(Collectors.toList());
    List<TermedStatementEntityEdit> batch = qids.stream().map(qid -> new TermedStatementEntityEditBuilder(qid).addDescription(description, true).build()).collect(Collectors.toList());
    int batchSize = 50;
    List<ItemDocument> fullBatch = qids.stream().map(qid -> ItemDocumentBuilder.forItemId(qid).withStatement(TestingData.generateStatement(qid, TestingData.existingId)).build()).collect(Collectors.toList());
    List<ItemDocument> firstBatch = fullBatch.subList(0, batchSize);
    List<ItemDocument> secondBatch = fullBatch.subList(batchSize, fullBatch.size());
    when(fetcher.getEntityDocuments(toQids(firstBatch))).thenReturn(toMap(firstBatch));
    when(fetcher.getEntityDocuments(toQids(secondBatch))).thenReturn(toMap(secondBatch));
    // Run edits
    EditBatchProcessor processor = new EditBatchProcessor(fetcher, editor, batch, library, summary, maxlag, tags, batchSize);
    assertEquals(0, processor.progress());
    for (int i = 124; i < 190; i++) {
        assertEquals(processor.remainingEdits(), 190 - i);
        processor.performEdit();
    }
    assertEquals(0, processor.remainingEdits());
    assertEquals(100, processor.progress());
    // Check result
    assertEquals(new NewEntityLibrary(), library);
    verify(fetcher, times(1)).getEntityDocuments(toQids(firstBatch));
    verify(fetcher, times(1)).getEntityDocuments(toQids(secondBatch));
    for (ItemDocument doc : fullBatch) {
        verify(editor, times(1)).editEntityDocument(Datamodel.makeItemUpdate(doc.getEntityId(), doc.getRevisionId(), Datamodel.makeTermUpdate(Collections.emptyList(), Collections.emptyList()), Datamodel.makeTermUpdate(Collections.singletonList(description), Collections.emptyList()), Collections.emptyMap(), Datamodel.makeStatementUpdate(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()), Collections.emptyList(), Collections.emptyList()), false, summary, tags);
    }
}
Also used : Arrays(java.util.Arrays) Assert.assertEquals(org.testng.Assert.assertEquals) Test(org.testng.annotations.Test) MediaInfoDocument(org.wikidata.wdtk.datamodel.interfaces.MediaInfoDocument) EntityDocument(org.wikidata.wdtk.datamodel.interfaces.EntityDocument) ArrayList(java.util.ArrayList) TermUpdate(org.wikidata.wdtk.datamodel.interfaces.TermUpdate) WikibaseDataFetcher(org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher) ItemIdValue(org.wikidata.wdtk.datamodel.interfaces.ItemIdValue) Map(java.util.Map) MonolingualTextValue(org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue) WikidataRefineTest(org.openrefine.wikidata.testing.WikidataRefineTest) TermedStatementEntityEdit(org.openrefine.wikidata.updates.TermedStatementEntityEdit) BeforeMethod(org.testng.annotations.BeforeMethod) ItemDocumentBuilder(org.wikidata.wdtk.datamodel.helpers.ItemDocumentBuilder) WikibaseDataEditor(org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor) Mockito.times(org.mockito.Mockito.times) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) Collectors(java.util.stream.Collectors) MediaInfoIdValue(org.wikidata.wdtk.datamodel.interfaces.MediaInfoIdValue) Mockito.verify(org.mockito.Mockito.verify) StatementUpdate(org.wikidata.wdtk.datamodel.interfaces.StatementUpdate) List(java.util.List) TermedStatementEntityEditBuilder(org.openrefine.wikidata.updates.TermedStatementEntityEditBuilder) Datamodel(org.wikidata.wdtk.datamodel.helpers.Datamodel) ItemDocument(org.wikidata.wdtk.datamodel.interfaces.ItemDocument) MediaWikiApiErrorException(org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException) Collections(java.util.Collections) TestingData(org.openrefine.wikidata.testing.TestingData) Mockito.mock(org.mockito.Mockito.mock) TermedStatementEntityEditBuilder(org.openrefine.wikidata.updates.TermedStatementEntityEditBuilder) ArrayList(java.util.ArrayList) MonolingualTextValue(org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue) ItemIdValue(org.wikidata.wdtk.datamodel.interfaces.ItemIdValue) ItemDocument(org.wikidata.wdtk.datamodel.interfaces.ItemDocument) TermedStatementEntityEdit(org.openrefine.wikidata.updates.TermedStatementEntityEdit) Test(org.testng.annotations.Test) WikidataRefineTest(org.openrefine.wikidata.testing.WikidataRefineTest)

Example 2 with ItemDocument

use of org.wikidata.wdtk.datamodel.interfaces.ItemDocument in project OpenRefine by OpenRefine.

the class EditBatchProcessorTest method testNewItem.

@Test
public void testNewItem() throws InterruptedException, MediaWikiApiErrorException, IOException {
    List<TermedStatementEntityEdit> batch = new ArrayList<>();
    batch.add(new TermedStatementEntityEditBuilder(TestingData.existingId).addAlias(Datamodel.makeMonolingualTextValue("my new alias", "en")).addStatement(TestingData.generateStatementAddition(TestingData.existingId, TestingData.newIdA)).build());
    MonolingualTextValue label = Datamodel.makeMonolingualTextValue("better label", "en");
    batch.add(new TermedStatementEntityEditBuilder(TestingData.newIdA).addAlias(label).build());
    // Plan expected edits
    ItemDocument existingItem = ItemDocumentBuilder.forItemId(TestingData.existingId).withLabel(Datamodel.makeMonolingualTextValue("pomme", "fr")).withDescription(Datamodel.makeMonolingualTextValue("fruit délicieux", "fr")).build();
    when(fetcher.getEntityDocuments(Collections.singletonList(TestingData.existingId.getId()))).thenReturn(Collections.singletonMap(TestingData.existingId.getId(), existingItem));
    ItemDocument expectedNewItem = ItemDocumentBuilder.forItemId(TestingData.newIdA).withLabel(label).build();
    ItemDocument createdNewItem = ItemDocumentBuilder.forItemId(Datamodel.makeWikidataItemIdValue("Q1234")).withLabel(label).withRevisionId(37828L).build();
    when(editor.createItemDocument(expectedNewItem, summary, tags)).thenReturn(createdNewItem);
    EditBatchProcessor processor = new EditBatchProcessor(fetcher, editor, batch, library, summary, maxlag, tags, 50);
    assertEquals(2, processor.remainingEdits());
    assertEquals(0, processor.progress());
    processor.performEdit();
    assertEquals(1, processor.remainingEdits());
    assertEquals(50, processor.progress());
    processor.performEdit();
    assertEquals(0, processor.remainingEdits());
    assertEquals(100, processor.progress());
    // does not do anything
    processor.performEdit();
    assertEquals(0, processor.remainingEdits());
    assertEquals(100, processor.progress());
    NewEntityLibrary expectedLibrary = new NewEntityLibrary();
    expectedLibrary.setId(1234L, "Q1234");
    assertEquals(expectedLibrary, library);
}
Also used : TermedStatementEntityEditBuilder(org.openrefine.wikidata.updates.TermedStatementEntityEditBuilder) ItemDocument(org.wikidata.wdtk.datamodel.interfaces.ItemDocument) TermedStatementEntityEdit(org.openrefine.wikidata.updates.TermedStatementEntityEdit) ArrayList(java.util.ArrayList) MonolingualTextValue(org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue) Test(org.testng.annotations.Test) WikidataRefineTest(org.openrefine.wikidata.testing.WikidataRefineTest)

Example 3 with ItemDocument

use of org.wikidata.wdtk.datamodel.interfaces.ItemDocument in project OpenRefine by OpenRefine.

the class TermedStatementEntityEditTest method testToEntityUpdate.

@Test
public void testToEntityUpdate() {
    TermedStatementEntityEdit edit = new TermedStatementEntityEditBuilder(existingSubject).addAlias(Datamodel.makeMonolingualTextValue("alias", "en")).addStatement(statementUpdate1).build();
    ItemDocument itemDocument = ItemDocumentBuilder.forItemId(existingSubject).withStatement(statement2).withLabel(Datamodel.makeMonolingualTextValue("label", "en")).build();
    ItemUpdate update = (ItemUpdate) edit.toEntityUpdate(itemDocument);
    assertEquals(update.getEntityId(), existingSubject);
    assertEquals(update.getAliases(), Collections.singletonMap("en", Datamodel.makeAliasUpdate(Collections.singletonList(Datamodel.makeMonolingualTextValue("alias", "en")), Collections.emptyList())));
    assertTrue(update.getDescriptions().isEmpty());
    assertTrue(update.getLabels().isEmpty());
    assertEquals(update.getStatements(), Datamodel.makeStatementUpdate(Collections.singletonList(statement1), Collections.emptyList(), Collections.emptySet()));
}
Also used : ItemDocument(org.wikidata.wdtk.datamodel.interfaces.ItemDocument) ItemUpdate(org.wikidata.wdtk.datamodel.interfaces.ItemUpdate) Test(org.testng.annotations.Test)

Example 4 with ItemDocument

use of org.wikidata.wdtk.datamodel.interfaces.ItemDocument in project OpenRefine by OpenRefine.

the class TermedStatementEntityEditTest method testToNewEntity.

@Test
public void testToNewEntity() {
    TermedStatementEntityEdit edit = new TermedStatementEntityEditBuilder(newSubject).addLabel(Datamodel.makeMonolingualTextValue("fr", "bonjour"), true).addDescription(Datamodel.makeMonolingualTextValue("de", "Redewendung"), true).build();
    ItemDocument itemDocument = (ItemDocument) edit.toNewEntity();
    ItemDocument expected = ItemDocumentBuilder.forItemId(newSubject).withLabel(Datamodel.makeMonolingualTextValue("fr", "bonjour")).withDescription(Datamodel.makeMonolingualTextValue("de", "Redewendung")).build();
    assertEquals(itemDocument, expected);
}
Also used : ItemDocument(org.wikidata.wdtk.datamodel.interfaces.ItemDocument) Test(org.testng.annotations.Test)

Example 5 with ItemDocument

use of org.wikidata.wdtk.datamodel.interfaces.ItemDocument in project OpenRefine by OpenRefine.

the class TermedStatementEntityEdit method toEntityUpdate.

/**
 * In case the subject id is not new, returns the corresponding update given
 * the current state of the entity.
 */
public EntityUpdate toEntityUpdate(EntityDocument entityDocument) {
    Validate.isFalse(isNew(), "Cannot create a corresponding entity update for a creation of a new entity.");
    if (id instanceof ItemIdValue) {
        ItemDocument itemDocument = (ItemDocument) entityDocument;
        // Labels
        List<MonolingualTextValue> labels = getLabels().stream().collect(Collectors.toList());
        labels.addAll(getLabelsIfNew().stream().filter(label -> !itemDocument.getLabels().containsKey(label.getLanguageCode())).collect(Collectors.toList()));
        TermUpdate labelUpdate = Datamodel.makeTermUpdate(labels, Collections.emptyList());
        // Descriptions
        List<MonolingualTextValue> descriptions = getDescriptions().stream().collect(Collectors.toList());
        descriptions.addAll(getDescriptionsIfNew().stream().filter(desc -> !itemDocument.getDescriptions().containsKey(desc.getLanguageCode())).collect(Collectors.toList()));
        TermUpdate descriptionUpdate = Datamodel.makeTermUpdate(descriptions, Collections.emptyList());
        // Aliases
        Set<MonolingualTextValue> aliases = getAliases();
        Map<String, List<MonolingualTextValue>> aliasesMap = aliases.stream().collect(Collectors.groupingBy(MonolingualTextValue::getLanguageCode));
        Map<String, AliasUpdate> aliasMap = aliasesMap.entrySet().stream().collect(Collectors.toMap(Entry::getKey, e -> Datamodel.makeAliasUpdate(e.getValue(), Collections.emptyList())));
        // Statements
        StatementUpdate statementUpdate = toStatementUpdate(itemDocument);
        return Datamodel.makeItemUpdate((ItemIdValue) getEntityId(), entityDocument.getRevisionId(), labelUpdate, descriptionUpdate, aliasMap, statementUpdate, Collections.emptyList(), Collections.emptyList());
    } else if (id instanceof MediaInfoIdValue) {
        MediaInfoDocument mediaInfoDocument = (MediaInfoDocument) entityDocument;
        // Labels (captions)
        List<MonolingualTextValue> labels = getLabels().stream().collect(Collectors.toList());
        labels.addAll(getLabelsIfNew().stream().filter(label -> !mediaInfoDocument.getLabels().containsKey(label.getLanguageCode())).collect(Collectors.toList()));
        TermUpdate labelUpdate = Datamodel.makeTermUpdate(labels, Collections.emptyList());
        // Statements
        StatementUpdate statementUpdate = toStatementUpdate(mediaInfoDocument);
        return Datamodel.makeMediaInfoUpdate((MediaInfoIdValue) id, entityDocument.getRevisionId(), labelUpdate, statementUpdate);
    } else {
        throw new NotImplementedException("Editing entities of type " + id.getEntityType() + " is not supported yet.");
    }
}
Also used : JsonProperty(com.fasterxml.jackson.annotation.JsonProperty) StatementUpdateBuilder(org.wikidata.wdtk.datamodel.helpers.StatementUpdateBuilder) HashMap(java.util.HashMap) MediaInfoDocument(org.wikidata.wdtk.datamodel.interfaces.MediaInfoDocument) EntityDocument(org.wikidata.wdtk.datamodel.interfaces.EntityDocument) ArrayList(java.util.ArrayList) StatementGroup(org.wikidata.wdtk.datamodel.interfaces.StatementGroup) TermUpdate(org.wikidata.wdtk.datamodel.interfaces.TermUpdate) HashSet(java.util.HashSet) StatementEditingMode(org.openrefine.wikidata.schema.strategies.StatementEditingMode) TermedStatementDocument(org.wikidata.wdtk.datamodel.interfaces.TermedStatementDocument) ItemIdValue(org.wikidata.wdtk.datamodel.interfaces.ItemIdValue) Map(java.util.Map) NotImplementedException(org.apache.commons.lang.NotImplementedException) JsonIgnore(com.fasterxml.jackson.annotation.JsonIgnore) EntityUpdate(org.wikidata.wdtk.datamodel.interfaces.EntityUpdate) EntityIdValue(org.wikidata.wdtk.datamodel.interfaces.EntityIdValue) StatementDocument(org.wikidata.wdtk.datamodel.interfaces.StatementDocument) LinkedList(java.util.LinkedList) MonolingualTextValue(org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue) Validate(org.jsoup.helper.Validate) Collection(java.util.Collection) PropertyIdValue(org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue) Set(java.util.Set) Statement(org.wikidata.wdtk.datamodel.interfaces.Statement) Collectors(java.util.stream.Collectors) AliasUpdate(org.wikidata.wdtk.datamodel.interfaces.AliasUpdate) MediaInfoIdValue(org.wikidata.wdtk.datamodel.interfaces.MediaInfoIdValue) StatementUpdate(org.wikidata.wdtk.datamodel.interfaces.StatementUpdate) List(java.util.List) Datamodel(org.wikidata.wdtk.datamodel.helpers.Datamodel) Entry(java.util.Map.Entry) ItemDocument(org.wikidata.wdtk.datamodel.interfaces.ItemDocument) Collections(java.util.Collections) MediaInfoIdValue(org.wikidata.wdtk.datamodel.interfaces.MediaInfoIdValue) StatementUpdate(org.wikidata.wdtk.datamodel.interfaces.StatementUpdate) MediaInfoDocument(org.wikidata.wdtk.datamodel.interfaces.MediaInfoDocument) NotImplementedException(org.apache.commons.lang.NotImplementedException) MonolingualTextValue(org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue) ItemIdValue(org.wikidata.wdtk.datamodel.interfaces.ItemIdValue) ItemDocument(org.wikidata.wdtk.datamodel.interfaces.ItemDocument) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) List(java.util.List) AliasUpdate(org.wikidata.wdtk.datamodel.interfaces.AliasUpdate) TermUpdate(org.wikidata.wdtk.datamodel.interfaces.TermUpdate)

Aggregations

ItemDocument (org.wikidata.wdtk.datamodel.interfaces.ItemDocument)6 Test (org.testng.annotations.Test)4 ArrayList (java.util.ArrayList)3 TermedStatementEntityEdit (org.openrefine.wikidata.updates.TermedStatementEntityEdit)3 ItemIdValue (org.wikidata.wdtk.datamodel.interfaces.ItemIdValue)3 MediaInfoIdValue (org.wikidata.wdtk.datamodel.interfaces.MediaInfoIdValue)3 MonolingualTextValue (org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue)3 IOException (java.io.IOException)2 Collections (java.util.Collections)2 List (java.util.List)2 Map (java.util.Map)2 Collectors (java.util.stream.Collectors)2 NotImplementedException (org.apache.commons.lang.NotImplementedException)2 WikidataRefineTest (org.openrefine.wikidata.testing.WikidataRefineTest)2 TermedStatementEntityEditBuilder (org.openrefine.wikidata.updates.TermedStatementEntityEditBuilder)2 Datamodel (org.wikidata.wdtk.datamodel.helpers.Datamodel)2 EntityDocument (org.wikidata.wdtk.datamodel.interfaces.EntityDocument)2 EntityUpdate (org.wikidata.wdtk.datamodel.interfaces.EntityUpdate)2 MediaInfoDocument (org.wikidata.wdtk.datamodel.interfaces.MediaInfoDocument)2 StatementUpdate (org.wikidata.wdtk.datamodel.interfaces.StatementUpdate)2