Examples with MongoTypeStorage org.apache.rya.indexing.entity.storage.mongo.MongoTypeStorage used on opensource projects

Example 1 with MongoTypeStorage

use of org.apache.rya.indexing.entity.storage.mongo.MongoTypeStorage in project incubator-rya by apache.

the class DuplicateDataDetectorIT method testCreateEntityNearDuplicateConfigDisabled.

@Test
public void testCreateEntityNearDuplicateConfigDisabled() throws EntityStorageException, TypeStorageException, ConfigurationException, ObjectStorageException {
    // Create the types the Entity uses.
    final TypeStorage typeStorage = new MongoTypeStorage(super.getMongoClient(), RYA_INSTANCE_NAME);
    final Type personType = createPersonType();
    final Type employeeType = createEmployeeType();
    typeStorage.create(personType);
    typeStorage.create(employeeType);
    final Optional<Type> storedPersonType = typeStorage.get(personType.getId());
    final Optional<Type> storedEmployeeType = typeStorage.get(employeeType.getId());
    assertTrue(storedPersonType.isPresent());
    assertTrue(storedEmployeeType.isPresent());
    // Create it.
    final DuplicateDataConfig duplicateDataConfig = new DuplicateDataConfig(// boolean
    new Tolerance(0.0, ToleranceType.DIFFERENCE), // byte
    new Tolerance(0.0, ToleranceType.DIFFERENCE), // date
    new Tolerance(500.0, ToleranceType.DIFFERENCE), // double
    new Tolerance(0.0001, ToleranceType.PERCENTAGE), // float
    new Tolerance(0.0001, ToleranceType.PERCENTAGE), // integer
    new Tolerance(1.0, ToleranceType.DIFFERENCE), // long
    new Tolerance(1.0, ToleranceType.DIFFERENCE), // short
    new Tolerance(1.0, ToleranceType.DIFFERENCE), // string
    new Tolerance(1.0, ToleranceType.DIFFERENCE), // uri
    new Tolerance(1.0, ToleranceType.DIFFERENCE), new HashMap<String, List<String>>(), false);
    final DuplicateDataDetector duplicateDataDetector = new DuplicateDataDetector(duplicateDataConfig);
    final EntityStorage entityStorage = new MongoEntityStorage(super.getMongoClient(), RYA_INSTANCE_NAME, duplicateDataDetector);
    final Entity bobEntity = createBobEntity();
    entityStorage.create(bobEntity);
    assertTrue(entityStorage.get(bobEntity.getSubject()).isPresent());
    final Builder duplicateBobBuilder = Entity.builder(createBobEntity());
    duplicateBobBuilder.setSubject(createRyaUri("Robert"));
    // Modify a property for each type that is within tolerance
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_AGE, shortRyaType((short) 41)));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_WEIGHT, floatRyaType(250.76f)));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_HEIGHT, doubleRyaType(72.499)));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_INCOME, intRyaType(50001)));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_NUMBER_OF_CHILDREN, byteRyaType((byte) 2)));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_LICENSE_NUMBER, longRyaType(123456789013L)));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_DATE_OF_BIRTH, dateRyaType(new DateTime(NOW.getTime() - 1).minusYears(40))));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_EXPIRATION_DATE, dateRyaType(new Date(NOW.getTime() - 1))));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_GLASSES, booleanRyaType(true)));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_EMAIL_ADDRESS, uriRyaType(new URIImpl("mailto:bob.smitch01@gmail.com"))));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_ADDRESS, stringRyaType("124 Fake St. Washington, DC 20024")));
    duplicateBobBuilder.setProperty(EMPLOYEE_TYPE_URI, new Property(HAS_EXTENSION, shortRyaType((short) 556)));
    final Entity duplicateBobEntity = duplicateBobBuilder.build();
    // Data duplication detection is disabled so it will be created.
    try {
        entityStorage.create(duplicateBobEntity);
    } catch (final EntityNearDuplicateException e) {
        fail();
    }
    assertTrue(entityStorage.get(duplicateBobEntity.getSubject()).isPresent());
    final Builder notDuplicateBobBuilder = Entity.builder(createBobEntity());
    notDuplicateBobBuilder.setSubject(createRyaUri("Not Bob"));
    // Modify a property for each type that is within tolerance
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_AGE, shortRyaType((short) 50)));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_WEIGHT, floatRyaType(300.0f)));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_HEIGHT, doubleRyaType(100.0)));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_INCOME, intRyaType(60000)));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_NUMBER_OF_CHILDREN, byteRyaType((byte) 5)));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_LICENSE_NUMBER, longRyaType(9L)));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_DATE_OF_BIRTH, dateRyaType(new DateTime(NOW.getTime() - 10000000L).minusYears(40))));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_EXPIRATION_DATE, dateRyaType(new Date(NOW.getTime() - 10000000L))));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_GLASSES, booleanRyaType(false)));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_EMAIL_ADDRESS, uriRyaType(new URIImpl("mailto:bad.email.address@gmail.com"))));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_ADDRESS, stringRyaType("123456789 Fake St. Washington, DC 20024")));
    notDuplicateBobBuilder.setProperty(EMPLOYEE_TYPE_URI, new Property(HAS_EXTENSION, shortRyaType((short) 1000)));
    final Entity notDuplicateBobEntity = notDuplicateBobBuilder.build();
    // Data duplication detection is disabled so it will be created.
    try {
        entityStorage.create(notDuplicateBobEntity);
    } catch (final EntityNearDuplicateException e) {
        fail();
    }
    assertTrue(entityStorage.get(notDuplicateBobEntity.getSubject()).isPresent());
}

Example 2 with MongoTypeStorage

use of org.apache.rya.indexing.entity.storage.mongo.MongoTypeStorage in project incubator-rya by apache.

the class DuplicateDataDetectorIT method testCreateEntityNearDuplicate.

@Test
public void testCreateEntityNearDuplicate() throws EntityStorageException, TypeStorageException, ObjectStorageException {
    // Create the types the Entity uses.
    final TypeStorage typeStorage = new MongoTypeStorage(super.getMongoClient(), RYA_INSTANCE_NAME);
    final Type personType = createPersonType();
    final Type employeeType = createEmployeeType();
    typeStorage.create(personType);
    typeStorage.create(employeeType);
    final Optional<Type> storedPersonType = typeStorage.get(personType.getId());
    final Optional<Type> storedEmployeeType = typeStorage.get(employeeType.getId());
    assertTrue(storedPersonType.isPresent());
    assertTrue(storedEmployeeType.isPresent());
    // Create it.
    final DuplicateDataConfig duplicateDataConfig = new DuplicateDataConfig(// boolean
    new Tolerance(0.0, ToleranceType.DIFFERENCE), // byte
    new Tolerance(0.0, ToleranceType.DIFFERENCE), // date
    new Tolerance(500.0, ToleranceType.DIFFERENCE), // double
    new Tolerance(0.0001, ToleranceType.PERCENTAGE), // float
    new Tolerance(0.0001, ToleranceType.PERCENTAGE), // integer
    new Tolerance(1.0, ToleranceType.DIFFERENCE), // long
    new Tolerance(1.0, ToleranceType.DIFFERENCE), // short
    new Tolerance(1.0, ToleranceType.DIFFERENCE), // string
    new Tolerance(1.0, ToleranceType.DIFFERENCE), // uri
    new Tolerance(1.0, ToleranceType.DIFFERENCE), new HashMap<String, List<String>>(), true);
    final DuplicateDataDetector duplicateDataDetector = new DuplicateDataDetector(duplicateDataConfig);
    final EntityStorage entityStorage = new MongoEntityStorage(super.getMongoClient(), RYA_INSTANCE_NAME, duplicateDataDetector);
    final Entity bobEntity = createBobEntity();
    entityStorage.create(bobEntity);
    assertTrue(entityStorage.get(bobEntity.getSubject()).isPresent());
    final Builder duplicateBobBuilder = Entity.builder(createBobEntity());
    duplicateBobBuilder.setSubject(createRyaUri("Robert"));
    // Modify a property for each type that is within tolerance
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_AGE, shortRyaType((short) 41)));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_WEIGHT, floatRyaType(250.76f)));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_HEIGHT, doubleRyaType(72.499)));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_INCOME, intRyaType(50001)));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_NUMBER_OF_CHILDREN, byteRyaType((byte) 2)));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_LICENSE_NUMBER, longRyaType(123456789013L)));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_DATE_OF_BIRTH, dateRyaType(new DateTime(NOW.getTime() - 1).minusYears(40))));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_EXPIRATION_DATE, dateRyaType(new Date(NOW.getTime() - 1))));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_GLASSES, booleanRyaType(true)));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_EMAIL_ADDRESS, uriRyaType(new URIImpl("mailto:bob.smitch01@gmail.com"))));
    duplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_ADDRESS, stringRyaType("124 Fake St. Washington, DC 20024")));
    duplicateBobBuilder.setProperty(EMPLOYEE_TYPE_URI, new Property(HAS_EXTENSION, shortRyaType((short) 556)));
    final Entity duplicateBobEntity = duplicateBobBuilder.build();
    // Try to create another entity that's considered a duplicate.
    // It will NOT be be created.
    boolean hasDuplicate = false;
    try {
        entityStorage.create(duplicateBobEntity);
    } catch (final EntityNearDuplicateException e) {
        hasDuplicate = true;
    }
    assertTrue(hasDuplicate);
    assertFalse(entityStorage.get(duplicateBobEntity.getSubject()).isPresent());
    final Builder notDuplicateBobBuilder = Entity.builder(createBobEntity());
    notDuplicateBobBuilder.setSubject(createRyaUri("Not Bob"));
    // Modify a property for each type that is within tolerance
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_AGE, shortRyaType((short) 50)));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_WEIGHT, floatRyaType(300.0f)));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_HEIGHT, doubleRyaType(100.0)));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_INCOME, intRyaType(60000)));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_NUMBER_OF_CHILDREN, byteRyaType((byte) 5)));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_LICENSE_NUMBER, longRyaType(9L)));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_DATE_OF_BIRTH, dateRyaType(new DateTime(NOW.getTime() - 10000000L).minusYears(40))));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_EXPIRATION_DATE, dateRyaType(new Date(NOW.getTime() - 10000000L))));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_GLASSES, booleanRyaType(false)));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_EMAIL_ADDRESS, uriRyaType(new URIImpl("mailto:bad.email.address@gmail.com"))));
    notDuplicateBobBuilder.setProperty(PERSON_TYPE_URI, new Property(HAS_ADDRESS, stringRyaType("123456789 Fake St. Washington, DC 20024")));
    notDuplicateBobBuilder.setProperty(EMPLOYEE_TYPE_URI, new Property(HAS_EXTENSION, shortRyaType((short) 1000)));
    final Entity notDuplicateBobEntity = notDuplicateBobBuilder.build();
    // It will be created.
    try {
        entityStorage.create(notDuplicateBobEntity);
    } catch (final EntityNearDuplicateException e) {
        fail();
    }
    assertTrue(entityStorage.get(notDuplicateBobEntity.getSubject()).isPresent());
}

Example 3 with MongoTypeStorage

use of org.apache.rya.indexing.entity.storage.mongo.MongoTypeStorage in project incubator-rya by apache.

the class MongoEntityIndexerIT method deleteStatement_deletesType.

@Test
public void deleteStatement_deletesType() throws Exception {
    try (MongoEntityIndexer indexer = new MongoEntityIndexer()) {
        indexer.setConf(conf);
        indexer.init();
        // Load the type into the TypeStorage.
        final TypeStorage types = new MongoTypeStorage(getMongoClient(), conf.getRyaInstanceName());
        types.create(PERSON_TYPE);
        types.create(EMPLOYEE_TYPE);
        // Index a bunch of RyaStatements.
        final RyaURI aliceSSN = new RyaURI("urn:SSN/111-11-1111");
        indexer.storeStatements(Sets.newHashSet(new RyaStatement(aliceSSN, new RyaURI(RDF.TYPE.toString()), new RyaType(XMLSchema.ANYURI, "urn:person")), new RyaStatement(aliceSSN, new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Alice")), new RyaStatement(aliceSSN, new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30")), new RyaStatement(aliceSSN, new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue"))));
        // Remove the explicit type from Alice.
        indexer.deleteStatement(new RyaStatement(aliceSSN, new RyaURI(RDF.TYPE.toString()), new RyaType(XMLSchema.ANYURI, "urn:person")));
        // Fetch the Entity from storage and ensure it looks correct.
        final EntityStorage entities = new MongoEntityStorage(getMongoClient(), conf.getRyaInstanceName());
        final Entity entity = entities.get(new RyaURI("urn:SSN/111-11-1111")).get();
        final Entity expected = Entity.builder().setSubject(aliceSSN).setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Alice"))).setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30"))).setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue"))).setProperty(new RyaURI("urn:employee"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Alice"))).setVersion(entity.getVersion()).build();
        assertEquals(expected, entity);
    }
}

Example 4 with MongoTypeStorage

use of org.apache.rya.indexing.entity.storage.mongo.MongoTypeStorage in project incubator-rya by apache.

the class MongoEntityIndexerIT method addStatement_manyUpdates.

@Test
public void addStatement_manyUpdates() throws Exception {
    try (MongoEntityIndexer indexer = new MongoEntityIndexer()) {
        indexer.setConf(conf);
        indexer.init();
        // Load the types into the TypeStorage.
        final TypeStorage types = new MongoTypeStorage(getMongoClient(), conf.getRyaInstanceName());
        types.create(PERSON_TYPE);
        types.create(EMPLOYEE_TYPE);
        // Index a bunch of RyaStatements.
        final RyaURI aliceSSN = new RyaURI("urn:SSN/111-11-1111");
        indexer.storeStatement(new RyaStatement(aliceSSN, new RyaURI(RDF.TYPE.toString()), new RyaType(XMLSchema.ANYURI, "urn:person")));
        indexer.storeStatement(new RyaStatement(aliceSSN, new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Alice")));
        indexer.storeStatement(new RyaStatement(aliceSSN, new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30")));
        indexer.storeStatement(new RyaStatement(aliceSSN, new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue")));
        // Fetch the Entity from storage and ensure it looks correct.
        final EntityStorage entities = new MongoEntityStorage(getMongoClient(), conf.getRyaInstanceName());
        final Entity entity = entities.get(new RyaURI("urn:SSN/111-11-1111")).get();
        final Entity expected = Entity.builder().setSubject(aliceSSN).setExplicitType(new RyaURI("urn:person")).setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Alice"))).setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30"))).setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue"))).setProperty(new RyaURI("urn:employee"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Alice"))).setVersion(entity.getVersion()).build();
        assertEquals(expected, entity);
    }
}

Example 5 with MongoTypeStorage

use of org.apache.rya.indexing.entity.storage.mongo.MongoTypeStorage in project incubator-rya by apache.

the class MongoEntityIndexerIT method addStatements.

@Test
public void addStatements() throws Exception {
    try (MongoEntityIndexer indexer = new MongoEntityIndexer()) {
        indexer.setConf(conf);
        indexer.init();
        // Load the types into the TypeStorage.
        final TypeStorage types = new MongoTypeStorage(getMongoClient(), conf.getRyaInstanceName());
        types.create(PERSON_TYPE);
        types.create(EMPLOYEE_TYPE);
        // Index a bunch of RyaStatements.
        final RyaURI aliceSSN = new RyaURI("urn:SSN/111-11-1111");
        final RyaURI bobSSN = new RyaURI("urn:SSN/222-22-2222");
        indexer.storeStatements(Sets.newHashSet(new RyaStatement(aliceSSN, new RyaURI(RDF.TYPE.toString()), new RyaType(XMLSchema.ANYURI, "urn:person")), new RyaStatement(aliceSSN, new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Alice")), new RyaStatement(aliceSSN, new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30")), new RyaStatement(aliceSSN, new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue")), new RyaStatement(bobSSN, new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Bob")), new RyaStatement(bobSSN, new RyaURI("urn:hoursPerWeek"), new RyaType(XMLSchema.INT, "40")), new RyaStatement(bobSSN, new RyaURI(RDF.TYPE.toString()), new RyaType(XMLSchema.ANYURI, "urn:employee"))));
        // Fetch the Entity from storage and ensure it looks correct.
        final EntityStorage entities = new MongoEntityStorage(getMongoClient(), conf.getRyaInstanceName());
        final Entity alice = entities.get(aliceSSN).get();
        final Entity bob = entities.get(bobSSN).get();
        final Set<Entity> storedEntities = Sets.newHashSet(alice, bob);
        final Entity expectedAlice = Entity.builder().setSubject(aliceSSN).setExplicitType(new RyaURI("urn:person")).setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Alice"))).setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30"))).setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue"))).setProperty(new RyaURI("urn:employee"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Alice"))).setVersion(alice.getVersion()).build();
        final Entity expectedBob = Entity.builder().setSubject(bobSSN).setExplicitType(new RyaURI("urn:employee")).setProperty(new RyaURI("urn:employee"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Bob"))).setProperty(new RyaURI("urn:employee"), new Property(new RyaURI("urn:hoursPerWeek"), new RyaType(XMLSchema.INT, "40"))).setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Bob"))).setVersion(bob.getVersion()).build();
        final Set<Entity> expected = Sets.newHashSet(expectedAlice, expectedBob);
        assertEquals(expected, storedEntities);
    }
}