Examples with MongoDbBatchWriterConfig org.apache.rya.mongodb.batch.MongoDbBatchWriterConfig used on opensource projects

Example 1 with MongoDbBatchWriterConfig

use of org.apache.rya.mongodb.batch.MongoDbBatchWriterConfig in project incubator-rya by apache.

the class AbstractMongoIndexer method initCore.

protected void initCore() {
    dbName = conf.getMongoDBName();
    this.mongoClient = conf.getMongoClient();
    db = this.mongoClient.getDB(dbName);
    final String collectionName = conf.get(MongoDBRdfConfiguration.MONGO_COLLECTION_PREFIX, "rya") + getCollectionName();
    collection = db.getCollection(collectionName);
    flushEachUpdate = ((MongoDBRdfConfiguration) conf).flushEachUpdate();
    final MongoDbBatchWriterConfig mongoDbBatchWriterConfig = MongoDbBatchWriterUtils.getMongoDbBatchWriterConfig(conf);
    mongoDbBatchWriter = new MongoDbBatchWriter<>(new DbCollectionType(collection), mongoDbBatchWriterConfig);
    try {
        mongoDbBatchWriter.start();
    } catch (final MongoDbBatchWriterException e) {
        LOG.error("Error start MongoDB batch writer", e);
    }
}

Example 2 with MongoDbBatchWriterConfig

use of org.apache.rya.mongodb.batch.MongoDbBatchWriterConfig in project incubator-rya by apache.

the class MongoDBRyaBatchWriterIT method testMongoCollectionFlush.

@Test
public void testMongoCollectionFlush() throws Exception {
    final MongoDBStorageStrategy<RyaStatement> storageStrategy = new SimpleMongoDBStorageStrategy();
    final List<Document> documents = Lists.newArrayList(toDocument(storageStrategy.serialize(statement(1))), toDocument(storageStrategy.serialize(statement(2))), toDocument(storageStrategy.serialize(statement(2))), null, toDocument(storageStrategy.serialize(statement(3))), toDocument(storageStrategy.serialize(statement(3))), toDocument(storageStrategy.serialize(statement(4))));
    final MongoCollectionType mongoCollectionType = new MongoCollectionType(getRyaCollection());
    final MongoDbBatchWriterConfig mongoDbBatchWriterConfig = MongoDbBatchWriterUtils.getMongoDbBatchWriterConfig(conf);
    final MongoDbBatchWriter<Document> mongoDbBatchWriter = new MongoDbBatchWriter<>(mongoCollectionType, mongoDbBatchWriterConfig);
    mongoDbBatchWriter.start();
    mongoDbBatchWriter.addObjectsToQueue(documents);
    mongoDbBatchWriter.flush();
    Thread.sleep(1_000);
    mongoDbBatchWriter.addObjectsToQueue(documents);
    mongoDbBatchWriter.flush();
    Thread.sleep(1_000);
    mongoDbBatchWriter.shutdown();
    assertEquals(4, getRyaCollection().count());
}

Example 3 with MongoDbBatchWriterConfig

use of org.apache.rya.mongodb.batch.MongoDbBatchWriterConfig in project incubator-rya by apache.

the class MongoDBRyaBatchWriterIT method testDbCollectionFlush.

@Test
public void testDbCollectionFlush() throws Exception {
    final MongoDBStorageStrategy<RyaStatement> storageStrategy = new SimpleMongoDBStorageStrategy();
    final List<DBObject> objects = Lists.newArrayList(storageStrategy.serialize(statement(1)), storageStrategy.serialize(statement(2)), storageStrategy.serialize(statement(2)), null, storageStrategy.serialize(statement(3)), storageStrategy.serialize(statement(3)), storageStrategy.serialize(statement(4)));
    final DbCollectionType collectionType = new DbCollectionType(getRyaDbCollection());
    final MongoDbBatchWriterConfig mongoDbBatchWriterConfig = MongoDbBatchWriterUtils.getMongoDbBatchWriterConfig(conf);
    final MongoDbBatchWriter<DBObject> mongoDbBatchWriter = new MongoDbBatchWriter<>(collectionType, mongoDbBatchWriterConfig);
    mongoDbBatchWriter.start();
    mongoDbBatchWriter.addObjectsToQueue(objects);
    mongoDbBatchWriter.flush();
    Thread.sleep(1_000);
    mongoDbBatchWriter.addObjectsToQueue(objects);
    mongoDbBatchWriter.flush();
    Thread.sleep(1_000);
    mongoDbBatchWriter.shutdown();
    assertEquals(4, getRyaDbCollection().count());
}

Example 4 with MongoDbBatchWriterConfig

use of org.apache.rya.mongodb.batch.MongoDbBatchWriterConfig in project incubator-rya by apache.

the class MongoDBRyaDAO method init.

@Override
public void init() throws RyaDAOException {
    if (isInitialized.get()) {
        return;
    }
    secondaryIndexers = conf.getAdditionalIndexers();
    for (final MongoSecondaryIndex index : secondaryIndexers) {
        index.setConf(conf);
    }
    db = mongoClient.getDB(conf.get(MongoDBRdfConfiguration.MONGO_DB_NAME));
    coll = db.getCollection(conf.getTriplesCollectionName());
    nameSpaceManager = new SimpleMongoDBNamespaceManager(db.getCollection(conf.getNameSpacesCollectionName()));
    queryEngine = new MongoDBQueryEngine();
    queryEngine.setConf(conf);
    storageStrategy = new SimpleMongoDBStorageStrategy();
    storageStrategy.createIndices(coll);
    for (final MongoSecondaryIndex index : secondaryIndexers) {
        index.init();
    }
    final MongoDbBatchWriterConfig mongoDbBatchWriterConfig = MongoDbBatchWriterUtils.getMongoDbBatchWriterConfig(conf);
    mongoDbBatchWriter = new MongoDbBatchWriter<>(new DbCollectionType(coll), mongoDbBatchWriterConfig);
    try {
        mongoDbBatchWriter.start();
    } catch (final MongoDbBatchWriterException e) {
        throw new RyaDAOException("Error starting MongoDB batch writer", e);
    }
    isInitialized.set(true);
}