Examples with EntityDuplicateTracker org.bboxdb.storage.util.EntityDuplicateTracker used on opensource projects

Example 1 with EntityDuplicateTracker

use of org.bboxdb.storage.util.EntityDuplicateTracker in project bboxdb by jnidzwetzki.

the class JoinedTupleListFuture method createSimpleIterator.

@Override
protected Iterator<JoinedTuple> createSimpleIterator() {
    final List<JoinedTuple> allTuples = getListWithAllResults();
    final EntityDuplicateTracker entityDuplicateTracker = new EntityDuplicateTracker();
    final Iterator<JoinedTuple> iterator = allTuples.iterator();
    while (iterator.hasNext()) {
        final JoinedTuple nextElement = iterator.next();
        if (entityDuplicateTracker.isElementAlreadySeen(nextElement)) {
            iterator.remove();
        }
    }
    return allTuples.iterator();
}

Example 2 with EntityDuplicateTracker

use of org.bboxdb.storage.util.EntityDuplicateTracker in project bboxdb by jnidzwetzki.

the class TestTupleDuplicateRemover method testTupleDuplicateRemoverEqualKeyDiffVesion.

@Test(timeout = 60000)
public void testTupleDuplicateRemoverEqualKeyDiffVesion() {
    final EntityDuplicateTracker tupleDuplicateRemover = new EntityDuplicateTracker();
    final Tuple tuple1 = new Tuple("key1", BoundingBox.FULL_SPACE, "".getBytes(), 1);
    Assert.assertFalse(tupleDuplicateRemover.isElementAlreadySeen(tuple1));
    Assert.assertTrue(tupleDuplicateRemover.isElementAlreadySeen(tuple1));
    final Tuple tuple2 = new Tuple("key1", BoundingBox.FULL_SPACE, "".getBytes(), 2);
    Assert.assertFalse(tupleDuplicateRemover.isElementAlreadySeen(tuple2));
    Assert.assertTrue(tupleDuplicateRemover.isElementAlreadySeen(tuple2));
}

Example 3 with EntityDuplicateTracker

use of org.bboxdb.storage.util.EntityDuplicateTracker in project bboxdb by jnidzwetzki.

the class TupleListFuture method createSimpleIterator.

/**
 * Returns a simple iterator, used for non paged results
 * @return
 */
@Override
protected Iterator<Tuple> createSimpleIterator() {
    final List<Tuple> allTuples = getListWithAllResults();
    // Sort tuples
    allTuples.sort(TupleHelper.TUPLE_KEY_AND_VERSION_COMPARATOR);
    // Remove duplicates
    duplicateResolver.removeDuplicates(allTuples);
    // Perform read repair
    performReadRepair(allTuples);
    final EntityDuplicateTracker entityDuplicateTracker = new EntityDuplicateTracker();
    final Iterator<Tuple> iterator = allTuples.iterator();
    while (iterator.hasNext()) {
        final Tuple nextElement = iterator.next();
        if (entityDuplicateTracker.isElementAlreadySeen(nextElement)) {
            iterator.remove();
        }
    }
    return allTuples.iterator();
}

Example 4 with EntityDuplicateTracker

use of org.bboxdb.storage.util.EntityDuplicateTracker in project bboxdb by jnidzwetzki.

the class TestTupleDuplicateRemover method testTupleDuplicateRemoverEqualKeyEqualVesion.

@Test(timeout = 60000)
public void testTupleDuplicateRemoverEqualKeyEqualVesion() {
    final EntityDuplicateTracker tupleDuplicateRemover = new EntityDuplicateTracker();
    final Tuple tuple1 = new Tuple("key1", BoundingBox.FULL_SPACE, "".getBytes(), 1);
    Assert.assertFalse(tupleDuplicateRemover.isElementAlreadySeen(tuple1));
    Assert.assertTrue(tupleDuplicateRemover.isElementAlreadySeen(tuple1));
    final Tuple tuple2 = new Tuple("key1", BoundingBox.FULL_SPACE, "".getBytes(), 1);
    Assert.assertTrue(tupleDuplicateRemover.isElementAlreadySeen(tuple2));
    Assert.assertTrue(tupleDuplicateRemover.isElementAlreadySeen(tuple2));
}

Example 5 with EntityDuplicateTracker

use of org.bboxdb.storage.util.EntityDuplicateTracker in project bboxdb by jnidzwetzki.

the class TestTupleDuplicateRemover method testJoinedTuple.

@Test(timeout = 60000)
public void testJoinedTuple() {
    final EntityDuplicateTracker tupleDuplicateRemover = new EntityDuplicateTracker();
    final Tuple tuple1 = new Tuple("key1", BoundingBox.FULL_SPACE, "".getBytes(), 1);
    final Tuple tuple2 = new Tuple("key2", BoundingBox.FULL_SPACE, "".getBytes(), 1);
    final Tuple tuple3 = new Tuple("key2", BoundingBox.FULL_SPACE, "".getBytes(), 1);
    final List<Tuple> tupleList1 = Arrays.asList(tuple1);
    final List<Tuple> tupleList2 = Arrays.asList(tuple1, tuple2);
    final List<Tuple> tupleList3 = Arrays.asList(tuple1, tuple2, tuple3);
    final JoinedTuple joinedTuple1 = new JoinedTuple(tupleList1, Arrays.asList("table1"));
    final JoinedTuple joinedTuple2 = new JoinedTuple(tupleList2, Arrays.asList("table1", "table2"));
    final JoinedTuple joinedTuple3 = new JoinedTuple(tupleList3, Arrays.asList("table1", "table2", "table3"));
    Assert.assertFalse(tupleDuplicateRemover.isElementAlreadySeen(joinedTuple1));
    Assert.assertTrue(tupleDuplicateRemover.isElementAlreadySeen(joinedTuple1));
    Assert.assertFalse(tupleDuplicateRemover.isElementAlreadySeen(joinedTuple2));
    Assert.assertTrue(tupleDuplicateRemover.isElementAlreadySeen(joinedTuple2));
    Assert.assertFalse(tupleDuplicateRemover.isElementAlreadySeen(joinedTuple3));
    Assert.assertTrue(tupleDuplicateRemover.isElementAlreadySeen(joinedTuple3));
}