Examples with MongoCollection com.mongodb.client.MongoCollection used on opensource projects

Example 1 with MongoCollection

use of com.mongodb.client.MongoCollection in project DataX by alibaba.

the class Entry method doSplit.

public static List<Configuration> doSplit(Configuration originalSliceConfig, int adviceNumber, MongoClient mongoClient) {
    List<Configuration> confList = new ArrayList<Configuration>();
    String dbName = originalSliceConfig.getString(KeyConstant.MONGO_DB_NAME);
    String collectionName = originalSliceConfig.getString(KeyConstant.MONGO_COLLECTION_NAME);
    if (Strings.isNullOrEmpty(dbName) || Strings.isNullOrEmpty(collectionName) || mongoClient == null) {
        throw DataXException.asDataXException(MongoDBReaderErrorCode.ILLEGAL_VALUE, MongoDBReaderErrorCode.ILLEGAL_VALUE.getDescription());
    }
    String query = originalSliceConfig.getString(KeyConstant.MONGO_QUERY);
    MongoDatabase db = mongoClient.getDatabase(dbName);
    MongoCollection collection = db.getCollection(collectionName);
    List<Entry> countInterval = doSplitInterval(adviceNumber, collection, query);
    for (Entry interval : countInterval) {
        Configuration conf = originalSliceConfig.clone();
        conf.set(KeyConstant.SKIP_COUNT, interval.interval);
        conf.set(KeyConstant.BATCH_SIZE, interval.batchSize);
        confList.add(conf);
    }
    return confList;
}

Example 2 with MongoCollection

use of com.mongodb.client.MongoCollection in project beam by apache.

the class MongoDbIOTest method setup.

@Before
public void setup() throws Exception {
    LOG.info("Starting MongoDB embedded instance on {}", port);
    try {
        Files.forceDelete(new File(MONGODB_LOCATION));
    } catch (Exception e) {
    }
    new File(MONGODB_LOCATION).mkdirs();
    IMongodConfig mongodConfig = new MongodConfigBuilder().version(Version.Main.PRODUCTION).configServer(false).replication(new Storage(MONGODB_LOCATION, null, 0)).net(new Net("localhost", port, Network.localhostIsIPv6())).cmdOptions(new MongoCmdOptionsBuilder().syncDelay(10).useNoPrealloc(true).useSmallFiles(true).useNoJournal(true).build()).build();
    mongodExecutable = mongodStarter.prepare(mongodConfig);
    mongodProcess = mongodExecutable.start();
    LOG.info("Insert test data");
    MongoClient client = new MongoClient("localhost", port);
    MongoDatabase database = client.getDatabase(DATABASE);
    MongoCollection collection = database.getCollection(COLLECTION);
    String[] scientists = { "Einstein", "Darwin", "Copernicus", "Pasteur", "Curie", "Faraday", "Newton", "Bohr", "Galilei", "Maxwell" };
    for (int i = 1; i <= 1000; i++) {
        int index = i % scientists.length;
        Document document = new Document();
        document.append("_id", i);
        document.append("scientist", scientists[index]);
        collection.insertOne(document);
    }
}

Example 3 with MongoCollection

use of com.mongodb.client.MongoCollection in project jackrabbit-oak by apache.

the class MongoDocumentTraverser method getAllDocuments.

public <T extends Document> CloseableIterable<T> getAllDocuments(Collection<T> collection, Predicate<String> filter) {
    if (!disableReadOnlyCheck) {
        checkState(mongoStore.isReadOnly(), "Traverser can only be used with readOnly store");
    }
    MongoCollection<BasicDBObject> dbCollection = mongoStore.getDBCollection(collection);
    // TODO This may lead to reads being routed to secondary depending on MongoURI
    // So caller must ensure that its safe to read from secondary
    Iterable<BasicDBObject> cursor = dbCollection.withReadPreference(mongoStore.getConfiguredReadPreference(collection)).find();
    CloseableIterable<BasicDBObject> closeableCursor = CloseableIterable.wrap(cursor);
    cursor = closeableCursor;
    @SuppressWarnings("Guava") Iterable<T> result = FluentIterable.from(cursor).filter(o -> filter.test((String) o.get(Document.ID))).transform(o -> {
        T doc = mongoStore.convertFromDBObject(collection, o);
        // TODO Review the cache update approach where tracker has to track *all* docs
        if (collection == Collection.NODES) {
            NodeDocument nodeDoc = (NodeDocument) doc;
            getNodeDocCache().put(nodeDoc);
        }
        return doc;
    });
    return CloseableIterable.wrap(result, closeableCursor);
}

Example 4 with MongoCollection

use of com.mongodb.client.MongoCollection in project debezium by debezium.

the class ReplicatorIT method shouldReplicateContent.

@Test
public void shouldReplicateContent() throws InterruptedException {
    Testing.Print.disable();
    // Update the configuration to add a collection filter ...
    useConfiguration(config.edit().with(MongoDbConnectorConfig.MAX_FAILED_CONNECTIONS, 1).with(MongoDbConnectorConfig.COLLECTION_WHITELIST, "dbA.contacts").build());
    TestHelper.cleanDatabase(primary, "dbA");
    // ------------------------------------------------------------------------------
    // ADD A DOCUMENT
    // ------------------------------------------------------------------------------
    // Add a document to the 'contacts' database ...
    primary.execute("shouldCreateContactsDatabase", mongo -> {
        Testing.debug("Populating the 'dbA.contacts' collection");
        // Create a database and a collection in that database ...
        MongoDatabase db = mongo.getDatabase("dbA");
        MongoCollection<Document> contacts = db.getCollection("contacts");
        InsertOneOptions insertOptions = new InsertOneOptions().bypassDocumentValidation(true);
        contacts.insertOne(Document.parse("{ \"name\":\"Jon Snow\"}"), insertOptions);
        assertThat(db.getCollection("contacts").count()).isEqualTo(1);
        // Read the collection to make sure we can find our document ...
        Bson filter = Filters.eq("name", "Jon Snow");
        FindIterable<Document> movieResults = db.getCollection("contacts").find(filter);
        try (MongoCursor<Document> cursor = movieResults.iterator()) {
            assertThat(cursor.tryNext().getString("name")).isEqualTo("Jon Snow");
            assertThat(cursor.tryNext()).isNull();
        }
        Testing.debug("Completed document to 'dbA.contacts' collection");
    });
    // Start the replicator ...
    List<SourceRecord> records = new LinkedList<>();
    Replicator replicator = new Replicator(context, replicaSet, records::add, (x) -> {
    });
    Thread thread = new Thread(replicator::run);
    thread.start();
    // Sleep for 2 seconds ...
    Thread.sleep(2000);
    // ------------------------------------------------------------------------------
    // ADD A SECOND DOCUMENT
    // ------------------------------------------------------------------------------
    // Add more documents to the 'contacts' database ...
    final Object[] expectedNames = { "Jon Snow", "Sally Hamm" };
    primary.execute("shouldCreateContactsDatabase", mongo -> {
        Testing.debug("Populating the 'dbA.contacts' collection");
        // Create a database and a collection in that database ...
        MongoDatabase db = mongo.getDatabase("dbA");
        MongoCollection<Document> contacts = db.getCollection("contacts");
        InsertOneOptions insertOptions = new InsertOneOptions().bypassDocumentValidation(true);
        contacts.insertOne(Document.parse("{ \"name\":\"Sally Hamm\"}"), insertOptions);
        assertThat(db.getCollection("contacts").count()).isEqualTo(2);
        // Read the collection to make sure we can find our documents ...
        FindIterable<Document> movieResults = db.getCollection("contacts").find();
        Set<String> foundNames = new HashSet<>();
        try (MongoCursor<Document> cursor = movieResults.iterator()) {
            while (cursor.hasNext()) {
                String name = cursor.next().getString("name");
                foundNames.add(name);
            }
        }
        assertThat(foundNames).containsOnly(expectedNames);
        Testing.debug("Completed document to 'dbA.contacts' collection");
    });
    // For for a minimum number of events or max time ...
    // both documents
    int numEventsExpected = 2;
    long stop = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(3);
    while (records.size() < numEventsExpected && System.currentTimeMillis() < stop) {
        Thread.sleep(100);
    }
    // ------------------------------------------------------------------------------
    // STOP REPLICATOR AND VERIFY WE FOUND A TOTAL OF 2 EVENTS
    // ------------------------------------------------------------------------------
    replicator.stop();
    // Verify each record is valid and that we found the two records we expect ...
    final Set<String> foundNames = new HashSet<>();
    records.forEach(record -> {
        VerifyRecord.isValid(record);
        Struct value = (Struct) record.value();
        String after = value.getString("after");
        Document afterDoc = Document.parse(after);
        foundNames.add(afterDoc.getString("name"));
        Operation op = Operation.forCode(value.getString("op"));
        assertThat(op == Operation.READ || op == Operation.CREATE).isTrue();
    });
    assertThat(records.size()).isEqualTo(2);
    assertThat(foundNames).containsOnly(expectedNames);
    // ------------------------------------------------------------------------------
    // RESTART REPLICATOR FROM SAME POSITON
    // ------------------------------------------------------------------------------
    reuseConfiguration(config);
    // Start the replicator again ...
    records = new LinkedList<>();
    replicator = new Replicator(context, replicaSet, records::add, (x) -> {
    });
    thread = new Thread(replicator::run);
    thread.start();
    // Sleep for 2 seconds ...
    Thread.sleep(2000);
    // Stop the replicator ...
    replicator.stop();
    // We should not have found any new records ...
    records.forEach(record -> {
        VerifyRecord.isValid(record);
    });
    assertThat(records.isEmpty()).isTrue();
    // ------------------------------------------------------------------------------
    // START REPLICATOR AND ALSO REMOVE A DOCUMENT
    // ------------------------------------------------------------------------------
    // Update the configuration and don't use a collection filter ...
    reuseConfiguration(config.edit().with(MongoDbConnectorConfig.MAX_FAILED_CONNECTIONS, 1).build());
    // Start the replicator again ...
    records = new LinkedList<>();
    replicator = new Replicator(context, replicaSet, records::add, (x) -> {
    });
    thread = new Thread(replicator::run);
    thread.start();
    // Sleep for 2 seconds ...
    Thread.sleep(2000);
    // Remove Jon Snow ...
    AtomicReference<ObjectId> jonSnowId = new AtomicReference<>();
    primary.execute("removeJonSnow", mongo -> {
        MongoDatabase db = mongo.getDatabase("dbA");
        MongoCollection<Document> contacts = db.getCollection("contacts");
        // Read the collection to make sure we can find our document ...
        Bson filter = Filters.eq("name", "Jon Snow");
        FindIterable<Document> movieResults = db.getCollection("contacts").find(filter);
        try (MongoCursor<Document> cursor = movieResults.iterator()) {
            Document doc = cursor.tryNext();
            assertThat(doc.getString("name")).isEqualTo("Jon Snow");
            assertThat(cursor.tryNext()).isNull();
            jonSnowId.set(doc.getObjectId("_id"));
            assertThat(jonSnowId.get()).isNotNull();
        }
        // Remove the document by filter ...
        contacts.deleteOne(Filters.eq("name", "Jon Snow"));
        Testing.debug("Removed the Jon Snow document from 'dbA.contacts' collection");
    });
    // For for a minimum number of events or max time ...
    // just one delete event
    numEventsExpected = 1;
    stop = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(3);
    while (records.size() < numEventsExpected && System.currentTimeMillis() < stop) {
        Thread.sleep(100);
    }
    // Stop the replicator ...
    replicator.stop();
    // Verify each record is valid and that we found the one new DELETE record we expect ...
    Set<ObjectId> foundIds = new HashSet<>();
    records.forEach(record -> {
        VerifyRecord.isValid(record);
        Struct key = (Struct) record.key();
        ObjectId id = (ObjectId) (JSON.parse(key.getString("id")));
        foundIds.add(id);
        if (record.value() != null) {
            Struct value = (Struct) record.value();
            Operation op = Operation.forCode(value.getString("op"));
            assertThat(op).isEqualTo(Operation.DELETE);
        }
    });
    // 1 delete and 1 tombstone
    assertThat(records.size()).isEqualTo(2);
    // ------------------------------------------------------------------------------
    // START REPLICATOR TO PERFORM SNAPSHOT
    // ------------------------------------------------------------------------------
    // Update the configuration and don't use a collection filter ...
    useConfiguration(config);
    // Start the replicator again ...
    records = new LinkedList<>();
    replicator = new Replicator(context, replicaSet, records::add, (x) -> {
    });
    thread = new Thread(replicator::run);
    thread.start();
    // Sleep for 2 seconds ...
    Thread.sleep(2000);
    // Stop the replicator ...
    replicator.stop();
    // Verify each record is valid and that we found the two records we expect ...
    foundNames.clear();
    records.forEach(record -> {
        VerifyRecord.isValid(record);
        Struct value = (Struct) record.value();
        String after = value.getString("after");
        Document afterDoc = Document.parse(after);
        foundNames.add(afterDoc.getString("name"));
        Operation op = Operation.forCode(value.getString("op"));
        assertThat(op).isEqualTo(Operation.READ);
    });
    // We should not have found any new records ...
    assertThat(records.size()).isEqualTo(1);
    Object[] allExpectedNames = { "Sally Hamm" };
    assertThat(foundNames).containsOnly(allExpectedNames);
}

Example 5 with MongoCollection

use of com.mongodb.client.MongoCollection in project wildfly-swarm by wildfly-swarm.

the class StatefulTestBean method addProduct.

public String addProduct() {
    MongoCollection collection = null;
    Document query = null;
    try {
        collection = database.getCollection("company");
        String companyName = "Acme products";
        JsonObject object = Json.createObjectBuilder().add("companyName", companyName).add("street", "999 Flow Lane").add("city", "Indiville").add("_id", companyName).build();
        Document document = Document.parse(object.toString());
        collection.insertOne(document);
        query = new Document("_id", companyName);
        FindIterable cursor = collection.find(query);
        Object dbObject = cursor.first();
        return dbObject.toString();
    } finally {
        if (query != null) {
            collection.drop();
        }
    }
}