Examples with ChangeFetcher nl.knaw.huygens.timbuctoo.v5.dataset.ChangeFetcher used on opensource projects

Example 1 with ChangeFetcher

use of nl.knaw.huygens.timbuctoo.v5.dataset.ChangeFetcher in project timbuctoo by HuygensING.

the class BdbRmlDataSourceStore method onChangedSubject.

@Override
public void onChangedSubject(String subject, ChangeFetcher changeFetcher) throws RdfProcessingFailedException {
    try {
        boolean[] wasCollection = new boolean[] { false };
        StreamIterator.iterateAndCloseOrThrow(changeFetcher.getPredicates(subject, TIM_HAS_ROW, Direction.OUT, true, false, true), quad -> {
            wasCollection[0] = true;
            if (quad.getChangeType() == ChangeType.ASSERTED) {
                bdbWrapper.put(subject, quad.getObject());
            }
        });
        if (!wasCollection[0]) {
            try (Stream<CursorQuad> quads = changeFetcher.getPredicates(subject, TIM_HAS_ROW, Direction.IN, true, true, true)) {
                Optional<CursorQuad> isRawRow = quads.findFirst();
                if (isRawRow.isPresent()) {
                    final String collectionUri = isRawRow.get().getObject();
                    Map<String, Property> predicatesToStore = collectionProperties.computeIfAbsent(collectionUri, collection -> getPropertyNames(changeFetcher, collection));
                    switch(isRawRow.get().getChangeType()) {
                        case ASSERTED:
                            // add all unchanged and new predicates
                            StreamIterator.iterateAndCloseOrThrow(changeFetcher.getPredicates(subject, false, true, true), pred -> {
                                final Property property = predicatesToStore.get(pred.getPredicate());
                                if (property != null) {
                                    bdbWrapper.put(subject, property.newName + "\n" + pred.getObject());
                                }
                            });
                            break;
                        case RETRACTED:
                            // remove all unchanged and removed predicates
                            StreamIterator.iterateAndCloseOrThrow(changeFetcher.getPredicates(subject, true, true, false), pred -> {
                                final Property property = predicatesToStore.get(pred.getPredicate());
                                if (property != null) {
                                    bdbWrapper.delete(subject, property.oldName + "\n" + pred.getObject());
                                }
                            });
                            break;
                        case UNCHANGED:
                            // add all added predicates, remove all removed predicates
                            StreamIterator.iterateAndCloseOrThrow(changeFetcher.getPredicates(subject, true, false, true), pred -> {
                                final Property property = predicatesToStore.get(pred.getPredicate());
                                if (property != null) {
                                    if (pred.getChangeType() == ChangeType.RETRACTED) {
                                        bdbWrapper.delete(subject, property.oldName + "\n" + pred.getObject());
                                    } else {
                                        bdbWrapper.put(subject, property.oldName + "\n" + pred.getObject());
                                    }
                                }
                            });
                            break;
                        default:
                            throw new RuntimeException("Should not happen");
                    }
                }
            }
        }
    } catch (DatabaseWriteException e) {
        throw new RdfProcessingFailedException(e);
    }
}

Example 2 with ChangeFetcher

use of nl.knaw.huygens.timbuctoo.v5.dataset.ChangeFetcher in project timbuctoo by HuygensING.

the class BdbSchemaStore method updatePredicateType.

public void updatePredicateType(Type type, CursorQuad quad, boolean inc, ChangeFetcher changeFetcher) {
    final Predicate predicate = type.getOrCreatePredicate(quad.getPredicate(), quad.getDirection());
    if (quad.getValuetype().isPresent()) {
        predicate.incValueType(quad.getValuetype().get(), inc ? 1 : -1);
    } else {
        try (Stream<CursorQuad> typeQs = changeFetcher.getPredicates(quad.getObject(), RDF_TYPE, OUT, !inc, true, inc)) {
            boolean[] hadType = new boolean[] { false };
            typeQs.forEach(typeQ -> {
                hadType[0] = true;
                predicate.incReferenceType(typeQ.getObject(), inc ? 1 : -1);
            });
            if (!hadType[0]) {
                predicate.incReferenceType(UNKNOWN, inc ? 1 : -1);
            }
        }
    }
}

Example 3 with ChangeFetcher

use of nl.knaw.huygens.timbuctoo.v5.dataset.ChangeFetcher in project timbuctoo by HuygensING.

the class BdbSchemaStore method onChangedSubject.

@Override
public void onChangedSubject(String subject, ChangeFetcher changeFetcher) {
    // Step 1: Get the types that where added, unchanged, removed
    List<Type> addedTypes = new ArrayList<>();
    List<Type> removedTypes = new ArrayList<>();
    List<Type> unchangedTypes = new ArrayList<>();
    try (Stream<CursorQuad> subjTypes = changeFetcher.getPredicates(subject, RDF_TYPE, OUT, true, true, true)) {
        subjTypes.forEach(type -> {
            boolean hadTypesBefore = false;
            final Type typeOfSubject = types.computeIfAbsent(type.getObject(), TYPE_MAKER);
            if (type.getChangeType() == ChangeType.ASSERTED) {
                typeOfSubject.registerSubject(1);
                addedTypes.add(typeOfSubject);
            } else if (type.getChangeType() == ChangeType.RETRACTED) {
                hadTypesBefore = true;
                typeOfSubject.registerSubject(-1);
                removedTypes.add(typeOfSubject);
            } else if (type.getChangeType() == ChangeType.UNCHANGED) {
                hadTypesBefore = true;
                unchangedTypes.add(typeOfSubject);
            }
            if (!hadTypesBefore) {
                try (Stream<CursorQuad> predicates = changeFetcher.getPredicates(subject, true, true, false)) {
                    boolean subjectIsNew = !predicates.findAny().isPresent();
                    if (!subjectIsNew) {
                        final Type unknown = types.computeIfAbsent(UNKNOWN, TYPE_MAKER);
                        removedTypes.add(unknown);
                        unknown.registerSubject(-1);
                    }
                }
            }
        });
    }
    if (addedTypes.isEmpty() && unchangedTypes.isEmpty()) {
        // subject currently has no types
        if (removedTypes.isEmpty()) {
            // subject had no types either
            try (Stream<CursorQuad> predicates = changeFetcher.getPredicates(subject, true, true, false)) {
                boolean subjectIsNew = !predicates.findAny().isPresent();
                if (subjectIsNew) {
                    final Type unknown = types.computeIfAbsent(UNKNOWN, TYPE_MAKER);
                    addedTypes.add(unknown);
                    unknown.registerSubject(1);
                } else {
                    unchangedTypes.add(types.computeIfAbsent(UNKNOWN, TYPE_MAKER));
                }
            }
        } else {
            // subject has become unknown
            final Type unknown = types.computeIfAbsent(UNKNOWN, TYPE_MAKER);
            addedTypes.add(unknown);
            unknown.registerSubject(1);
        }
    }
    // it it was asserted -> add it to the unchanged types and to the added types
    try (Stream<CursorQuad> predicates = changeFetcher.getPredicates(subject, true, true, true)) {
        String prevPred = "";
        Direction[] prevDir = new Direction[] { null };
        int retractedCount = 0;
        int assertedCount = 0;
        int unchangedCount = 0;
        // updatePredicateOccurrence)
        for (CursorQuad quad : (Iterable<CursorQuad>) predicates::iterator) {
            boolean predicateSameAsPrev = prevPred.equals(quad.getPredicate()) && prevDir[0] == quad.getDirection();
            if (!predicateSameAsPrev) {
                updatePredicateOccurrence(addedTypes, removedTypes, unchangedTypes, retractedCount, unchangedCount, assertedCount, prevPred, prevDir[0]);
                prevPred = quad.getPredicate();
                prevDir[0] = quad.getDirection();
                retractedCount = 0;
                assertedCount = 0;
                unchangedCount = 0;
            }
            if (quad.getChangeType() == ChangeType.RETRACTED) {
                retractedCount++;
            } else if (quad.getChangeType() == ChangeType.UNCHANGED) {
                unchangedCount++;
            } else if (quad.getChangeType() == ChangeType.ASSERTED) {
                assertedCount++;
            }
            if (quad.getDirection() != Direction.IN) {
                if (quad.getChangeType() == ChangeType.RETRACTED) {
                    for (Type type : unchangedTypes) {
                        updatePredicateType(type, quad, false, changeFetcher);
                    }
                    for (Type type : removedTypes) {
                        updatePredicateType(type, quad, false, changeFetcher);
                    }
                } else if (quad.getChangeType() == ChangeType.UNCHANGED) {
                    for (Type type : removedTypes) {
                        updatePredicateType(type, quad, false, changeFetcher);
                    }
                    for (Type type : addedTypes) {
                        updatePredicateType(type, quad, true, changeFetcher);
                    }
                } else if (quad.getChangeType() == ChangeType.ASSERTED) {
                    for (Type type : unchangedTypes) {
                        updatePredicateType(type, quad, true, changeFetcher);
                    }
                    for (Type type : addedTypes) {
                        updatePredicateType(type, quad, true, changeFetcher);
                    }
                }
            }
        }
        updatePredicateOccurrence(addedTypes, removedTypes, unchangedTypes, retractedCount, unchangedCount, assertedCount, prevPred, prevDir[0]);
    }
}

Example 4 with ChangeFetcher

use of nl.knaw.huygens.timbuctoo.v5.dataset.ChangeFetcher in project timbuctoo by HuygensING.

the class StoreUpdater method updateListeners.

private void updateListeners() throws RdfProcessingFailedException {
    logString = "Processed {} subjects ({} subjects/s)";
    for (OptimizedPatchListener listener : listeners) {
        listener.start();
    }
    count = 0;
    prevCount = 0;
    prevTime = stopwatch.elapsed(TimeUnit.SECONDS);
    try (Stream<String> subjects = updatedPerPatchStore.ofVersion(currentversion)) {
        final ChangeFetcher getQuads = new ChangeFetcherImpl(truePatchStore, tripleStore, currentversion);
        final Iterator<String> iterator = subjects.iterator();
        while (iterator.hasNext()) {
            final boolean needUpdate = notifyUpdate();
            final String subject = iterator.next();
            for (OptimizedPatchListener listener : listeners) {
                if (needUpdate) {
                    listener.notifyUpdate();
                }
                listener.onChangedSubject(subject, getQuads);
            }
        }
    }
    for (OptimizedPatchListener listener : listeners) {
        listener.finish();
    }
}

Example 5 with ChangeFetcher

use of nl.knaw.huygens.timbuctoo.v5.dataset.ChangeFetcher in project timbuctoo by HuygensING.

the class BdbSchemaStoreTest method generatesASchemaForAllAdded.

@Test
public void generatesASchemaForAllAdded() throws Exception {
    final DummyDataStorage dataStore = new DummyDataStorage();
    ChangeFetcher changeFetcher = new DummyChangeFetcher(CursorQuad.create("subj", "pred", Direction.OUT, ChangeType.ASSERTED, "obj", null, null, ""), CursorQuad.create("subj", RdfConstants.RDF_TYPE, Direction.OUT, ChangeType.ASSERTED, "type", null, null, ""), CursorQuad.create("obj", "pred", Direction.IN, ChangeType.ASSERTED, "subj", null, null, ""));
    BdbSchemaStore schemaStore = new BdbSchemaStore(dataStore, new ImportStatus(new LogList()));
    schemaStore.start();
    schemaStore.onChangedSubject("subj", changeFetcher);
    schemaStore.onChangedSubject("obj", changeFetcher);
    schemaStore.finish();
    assertThat(dataStore.getResult(), is("{\n" + "  \"http://timbuctoo.huygens.knaw.nl/static/v5/vocabulary#unknown\" : {\n" + "    \"name\" : \"http://timbuctoo.huygens.knaw.nl/static/v5/vocabulary#unknown\",\n" + "    \"predicates\" : [ {\n" + "      \"name\" : \"pred\",\n" + "      \"direction\" : \"IN\",\n" + "      \"valueTypes\" : { },\n" + "      \"referenceTypes\" : {\n" + "        \"type\" : 1\n" + "      },\n" + "      \"subjectsWithThisPredicate\" : 1,\n" + "      \"subjectsWithThisPredicateAsList\" : 0,\n" + "      \"hasBeenList\" : false,\n" + "      \"hasBeenSingular\" : true\n" + "    } ],\n" + "    \"subjectsWithThisType\" : 1\n" + "  },\n" + "  \"type\" : {\n" + "    \"name\" : \"type\",\n" + "    \"predicates\" : [ {\n" + "      \"name\" : \"http://www.w3.org/1999/02/22-rdf-syntax-ns#type\",\n" + "      \"direction\" : \"OUT\",\n" + "      \"valueTypes\" : { },\n" + "      \"referenceTypes\" : {\n" + "        \"http://timbuctoo.huygens.knaw.nl/static/v5/vocabulary#unknown\" : 1\n" + "      },\n" + "      \"subjectsWithThisPredicate\" : 1,\n" + "      \"subjectsWithThisPredicateAsList\" : 0,\n" + "      \"hasBeenList\" : false,\n" + "      \"hasBeenSingular\" : true\n" + "    }, {\n" + "      \"name\" : \"pred\",\n" + "      \"direction\" : \"OUT\",\n" + "      \"valueTypes\" : { },\n" + "      \"referenceTypes\" : {\n" + "        \"http://timbuctoo.huygens.knaw.nl/static/v5/vocabulary#unknown\" : 1\n" + "      },\n" + "      \"subjectsWithThisPredicate\" : 1,\n" + "      \"subjectsWithThisPredicateAsList\" : 0,\n" + "      \"hasBeenList\" : false,\n" + "      \"hasBeenSingular\" : true\n" + "    } ],\n" + "    \"subjectsWithThisType\" : 1\n" + "  }\n" + "}"));
}