Search in sources :

Example 11 with Block

use of com.mongodb.Block in project mongo-java-driver by mongodb.

the class AbstractRetryableReadsTest method setUp.

@Before
public void setUp() {
    assumeFalse(skipTest);
    assumeTrue("Skipping test: " + definition.getString("skipReason", new BsonString("")).getValue(), !definition.containsKey("skipReason"));
    assumeFalse("Skipping count tests", filename.startsWith("count.") || filename.startsWith("count-"));
    assumeFalse("Skipping list index names tests", filename.startsWith("listIndexNames"));
    collectionHelper = new CollectionHelper<Document>(new DocumentCodec(), new MongoNamespace(databaseName, collectionName));
    final BsonDocument clientOptions = definition.getDocument("clientOptions", new BsonDocument());
    ConnectionString connectionString = getConnectionString();
    useMultipleMongoses = definition.getBoolean("useMultipleMongoses", BsonBoolean.FALSE).getValue();
    if (useMultipleMongoses) {
        assumeTrue(isSharded());
        connectionString = getMultiMongosConnectionString();
        assumeTrue("The system property org.mongodb.test.multi.mongos.uri is not set.", connectionString != null);
    }
    MongoClientSettings settings = getMongoClientSettingsBuilder().applyConnectionString(connectionString).addCommandListener(commandListener).applyToSocketSettings(new Block<SocketSettings.Builder>() {

        @Override
        public void apply(final SocketSettings.Builder builder) {
            builder.readTimeout(5, TimeUnit.SECONDS);
        }
    }).applyToServerSettings(new Block<ServerSettings.Builder>() {

        @Override
        public void apply(final ServerSettings.Builder builder) {
            builder.heartbeatFrequency(5, TimeUnit.MILLISECONDS);
        }
    }).writeConcern(getWriteConcern(clientOptions)).readConcern(getReadConcern(clientOptions)).readPreference(getReadPreference(clientOptions)).retryWrites(clientOptions.getBoolean("retryWrites", BsonBoolean.FALSE).getValue()).retryReads(clientOptions.getBoolean("retryReads", BsonBoolean.TRUE).getValue()).build();
    mongoClient = createMongoClient(settings);
    if (data != null) {
        List<BsonDocument> documents = new ArrayList<BsonDocument>();
        for (BsonValue document : data) {
            documents.add(document.asDocument());
        }
        collectionHelper.drop();
        if (documents.size() > 0) {
            collectionHelper.insertDocuments(documents);
        }
    }
    MongoDatabase database = mongoClient.getDatabase(databaseName);
    if (gridFSBucketName != null) {
        setupGridFSBuckets(database);
        commandListener.reset();
    }
    collection = database.getCollection(collectionName, BsonDocument.class);
    helper = new JsonPoweredCrudTestHelper(description, database, collection, gridFSBucket, mongoClient);
    if (definition.containsKey("failPoint")) {
        collectionHelper.runAdminCommand(definition.getDocument("failPoint"));
    }
}
Also used : Fixture.getMongoClientSettingsBuilder(com.mongodb.client.Fixture.getMongoClientSettingsBuilder) DocumentCodec(org.bson.codecs.DocumentCodec) BsonDocumentCodec(org.bson.codecs.BsonDocumentCodec) ArrayList(java.util.ArrayList) MongoClientSettings(com.mongodb.MongoClientSettings) Document(org.bson.Document) BsonDocument(org.bson.BsonDocument) MongoNamespace(com.mongodb.MongoNamespace) SocketSettings(com.mongodb.connection.SocketSettings) BsonDocument(org.bson.BsonDocument) BsonString(org.bson.BsonString) ServerSettings(com.mongodb.connection.ServerSettings) Block(com.mongodb.Block) ClusterFixture.getConnectionString(com.mongodb.ClusterFixture.getConnectionString) ClusterFixture.getMultiMongosConnectionString(com.mongodb.ClusterFixture.getMultiMongosConnectionString) ConnectionString(com.mongodb.ConnectionString) BsonValue(org.bson.BsonValue) Before(org.junit.Before)

Example 12 with Block

use of com.mongodb.Block in project repairnator by Spirals-Team.

the class ComputeTestFailingStats method main.

public static void main(String[] args) throws IOException {
    if (args.length < 4) {
        System.err.println("Usage: java ComputeTestFailingStats <url of mongodb with auth> <db name> <collection name> <path of the file to write>");
        System.exit(-1);
    }
    Map<String, Integer> occurencesByFailure = new HashMap<>();
    String dbCollectionUrl = args[0];
    String dbName = args[1];
    String collectionName = args[2];
    String pathOutput = args[3];
    File outputFile = new File(pathOutput);
    MongoConnection mongoConnection = new MongoConnection(dbCollectionUrl, dbName);
    MongoDatabase database = mongoConnection.getMongoDatabase();
    MongoCollection<Document> collection = database.getCollection(collectionName);
    Calendar limitDateFebruary2017 = Calendar.getInstance();
    Calendar limitDateJanuary2018 = Calendar.getInstance();
    limitDateFebruary2017.set(2017, Calendar.FEBRUARY, 1);
    limitDateJanuary2018.set(2018, Calendar.JANUARY, 1);
    Block<Document> block = new Block<Document>() {

        @Override
        public void apply(Document document) {
            totalFailingBuild++;
            String typeOfFailures = document.getString("typeOfFailures");
            for (String failure : typeOfFailures.split(",")) {
                if (failure.endsWith(":")) {
                    failure = failure.substring(0, failure.length() - 1);
                }
                if (failure.equals("skip") || failure.equals("skipped")) {
                    continue;
                }
                if (!occurencesByFailure.containsKey(failure)) {
                    occurencesByFailure.put(failure, 0);
                }
                int nbOcc = occurencesByFailure.get(failure);
                nbOcc++;
                occurencesByFailure.put(failure, nbOcc);
                totalNumberOfFailures++;
            }
        }
    };
    collection.find(and(lt("buildFinishedDate", limitDateJanuary2018.getTime()), gt("buildFinishedDate", limitDateFebruary2017.getTime()), in("status", "PATCHED", "test errors", "test failure"))).forEach(block);
    BufferedWriter buffer = new BufferedWriter(new FileWriter(outputFile));
    buffer.write("failure\tNb Occurences\n");
    buffer.flush();
    for (Map.Entry<String, Integer> entry : occurencesByFailure.entrySet().stream().sorted(Map.Entry.comparingByValue(Collections.reverseOrder())).collect(Collectors.toList())) {
        buffer.write(entry.getKey() + "\t" + entry.getValue() + "\n");
        buffer.flush();
    }
    buffer.close();
    System.out.println("Output written to " + pathOutput + " - " + totalFailingBuild + " failing build detected and " + totalNumberOfFailures + " failures counted.");
}
Also used : HashMap(java.util.HashMap) Calendar(java.util.Calendar) FileWriter(java.io.FileWriter) Document(org.bson.Document) BufferedWriter(java.io.BufferedWriter) Block(com.mongodb.Block) File(java.io.File) MongoConnection(fr.inria.spirals.repairnator.serializer.mongodb.MongoConnection) HashMap(java.util.HashMap) Map(java.util.Map) MongoDatabase(com.mongodb.client.MongoDatabase)

Example 13 with Block

use of com.mongodb.Block in project jackrabbit-oak by apache.

the class MongoDocumentStore method getModStamps.

/**
 * Returns the {@link Document#MOD_COUNT} and
 * {@link NodeDocument#MODIFIED_IN_SECS} values of the documents with the
 * given {@code keys}. The returned map will only contain entries for
 * existing documents. The default value is -1 if the document does not have
 * a modCount field. The same applies to the modified field.
 *
 * @param keys the keys of the documents.
 * @return map with key to modification stamp mapping.
 * @throws MongoException if the call fails
 */
@Nonnull
private Map<String, ModificationStamp> getModStamps(Iterable<String> keys) throws MongoException {
    // Fetch only the modCount and id
    final BasicDBObject fields = new BasicDBObject(Document.ID, 1);
    fields.put(Document.MOD_COUNT, 1);
    fields.put(NodeDocument.MODIFIED_IN_SECS, 1);
    Map<String, ModificationStamp> modCounts = Maps.newHashMap();
    nodes.withReadPreference(ReadPreference.primary()).find(Filters.in(Document.ID, keys)).projection(fields).forEach((Block<BasicDBObject>) obj -> {
        String id = (String) obj.get(Document.ID);
        Long modCount = Utils.asLong((Number) obj.get(Document.MOD_COUNT));
        if (modCount == null) {
            modCount = -1L;
        }
        Long modified = Utils.asLong((Number) obj.get(NodeDocument.MODIFIED_IN_SECS));
        if (modified == null) {
            modified = -1L;
        }
        modCounts.put(id, new ModificationStamp(modCount, modified));
    });
    return modCounts;
}
Also used : Arrays(java.util.Arrays) CacheStats(org.apache.jackrabbit.oak.cache.CacheStats) MongoDatabase(com.mongodb.client.MongoDatabase) Predicates.in(com.google.common.base.Predicates.in) Sets.difference(com.google.common.collect.Sets.difference) Key(org.apache.jackrabbit.oak.plugins.document.UpdateOp.Key) Predicates.not(com.google.common.base.Predicates.not) UpdateResult(com.mongodb.client.result.UpdateResult) Map(java.util.Map) UpdateOptions(com.mongodb.client.model.UpdateOptions) Revision(org.apache.jackrabbit.oak.plugins.document.Revision) CacheInvalidationStats(org.apache.jackrabbit.oak.plugins.document.cache.CacheInvalidationStats) Set(java.util.Set) Collection(org.apache.jackrabbit.oak.plugins.document.Collection) BulkWriteUpsert(com.mongodb.bulk.BulkWriteUpsert) Clock(org.apache.jackrabbit.oak.stats.Clock) CacheChangesTracker(org.apache.jackrabbit.oak.plugins.document.cache.CacheChangesTracker) Iterables.filter(com.google.common.collect.Iterables.filter) Iterables(com.google.common.collect.Iterables) MongoClientURI(com.mongodb.MongoClientURI) MongoCollection(com.mongodb.client.MongoCollection) BulkWriteOptions(com.mongodb.client.model.BulkWriteOptions) FindOneAndUpdateOptions(com.mongodb.client.model.FindOneAndUpdateOptions) Callable(java.util.concurrent.Callable) SD_MAX_REV_TIME_IN_SECS(org.apache.jackrabbit.oak.plugins.document.NodeDocument.SD_MAX_REV_TIME_IN_SECS) ArrayList(java.util.ArrayList) UpdateUtils(org.apache.jackrabbit.oak.plugins.document.UpdateUtils) LinkedHashMap(java.util.LinkedHashMap) Bson(org.bson.conversions.Bson) Filters(com.mongodb.client.model.Filters) Lists(com.google.common.collect.Lists) SD_TYPE(org.apache.jackrabbit.oak.plugins.document.NodeDocument.SD_TYPE) ReplicaSetInfo(org.apache.jackrabbit.oak.plugins.document.mongo.replica.ReplicaSetInfo) Nullable(javax.annotation.Nullable) DELETED_ONCE(org.apache.jackrabbit.oak.plugins.document.NodeDocument.DELETED_ONCE) Maps.filterKeys(com.google.common.collect.Maps.filterKeys) LocalChanges(org.apache.jackrabbit.oak.plugins.document.mongo.replica.LocalChanges) UpdateOneModel(com.mongodb.client.model.UpdateOneModel) BasicDBObject(com.mongodb.BasicDBObject) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) DocumentStoreStatsCollector(org.apache.jackrabbit.oak.plugins.document.DocumentStoreStatsCollector) Lock(java.util.concurrent.locks.Lock) TreeMap(java.util.TreeMap) StripedNodeDocumentLocks(org.apache.jackrabbit.oak.plugins.document.locks.StripedNodeDocumentLocks) RevisionListener(org.apache.jackrabbit.oak.plugins.document.RevisionListener) Condition.newEqualsCondition(org.apache.jackrabbit.oak.plugins.document.UpdateOp.Condition.newEqualsCondition) Date(java.util.Date) LoggerFactory(org.slf4j.LoggerFactory) MongoBulkWriteException(com.mongodb.MongoBulkWriteException) WriteModel(com.mongodb.client.model.WriteModel) Utils(org.apache.jackrabbit.oak.plugins.document.util.Utils) MongoUtils.createPartialIndex(org.apache.jackrabbit.oak.plugins.document.mongo.MongoUtils.createPartialIndex) DBObject(com.mongodb.DBObject) RevisionVector(org.apache.jackrabbit.oak.plugins.document.RevisionVector) MongoUtils.createIndex(org.apache.jackrabbit.oak.plugins.document.mongo.MongoUtils.createIndex) Function(com.google.common.base.Function) ImmutableMap(com.google.common.collect.ImmutableMap) MongoException(com.mongodb.MongoException) ReturnDocument(com.mongodb.client.model.ReturnDocument) PerfLogger(org.apache.jackrabbit.oak.commons.PerfLogger) NodeDocumentLocks(org.apache.jackrabbit.oak.plugins.document.locks.NodeDocumentLocks) List(java.util.List) DocumentStoreException.asDocumentStoreException(org.apache.jackrabbit.oak.plugins.document.DocumentStoreException.asDocumentStoreException) Block(com.mongodb.Block) JournalEntry(org.apache.jackrabbit.oak.plugins.document.JournalEntry) FindIterable(com.mongodb.client.FindIterable) MongoUtils.hasIndex(org.apache.jackrabbit.oak.plugins.document.mongo.MongoUtils.hasIndex) Entry(java.util.Map.Entry) NodeDocumentCache(org.apache.jackrabbit.oak.plugins.document.cache.NodeDocumentCache) Condition(org.apache.jackrabbit.oak.plugins.document.UpdateOp.Condition) ModificationStamp(org.apache.jackrabbit.oak.plugins.document.cache.ModificationStamp) ReadPreference(com.mongodb.ReadPreference) DocumentStoreException(org.apache.jackrabbit.oak.plugins.document.DocumentStoreException) Stopwatch(com.google.common.base.Stopwatch) MongoUtils.getDocumentStoreExceptionTypeFor(org.apache.jackrabbit.oak.plugins.document.mongo.MongoUtils.getDocumentStoreExceptionTypeFor) UpdateOp(org.apache.jackrabbit.oak.plugins.document.UpdateOp) HashMap(java.util.HashMap) Iterators(com.google.common.collect.Iterators) MODIFIED_IN_SECS(org.apache.jackrabbit.oak.plugins.document.NodeDocument.MODIFIED_IN_SECS) HashSet(java.util.HashSet) BulkWriteError(com.mongodb.bulk.BulkWriteError) ImmutableList(com.google.common.collect.ImmutableList) MongoCursor(com.mongodb.client.MongoCursor) UncheckedExecutionException(com.google.common.util.concurrent.UncheckedExecutionException) NodeDocument(org.apache.jackrabbit.oak.plugins.document.NodeDocument) Nonnull(javax.annotation.Nonnull) DocumentStore(org.apache.jackrabbit.oak.plugins.document.DocumentStore) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) Maps(com.google.common.collect.Maps) CacheValue(org.apache.jackrabbit.oak.cache.CacheValue) TimeUnit(java.util.concurrent.TimeUnit) StableRevisionComparator(org.apache.jackrabbit.oak.plugins.document.StableRevisionComparator) BulkWriteResult(com.mongodb.bulk.BulkWriteResult) MongoClient(com.mongodb.MongoClient) Document(org.apache.jackrabbit.oak.plugins.document.Document) Operation(org.apache.jackrabbit.oak.plugins.document.UpdateOp.Operation) WriteConcern(com.mongodb.WriteConcern) Collections(java.util.Collections) CheckForNull(javax.annotation.CheckForNull) BasicDBObject(com.mongodb.BasicDBObject) ModificationStamp(org.apache.jackrabbit.oak.plugins.document.cache.ModificationStamp) Nonnull(javax.annotation.Nonnull)

Example 14 with Block

use of com.mongodb.Block in project jackrabbit-oak by apache.

the class MongoVersionGCSupport method logSplitDocIdsTobeDeleted.

private void logSplitDocIdsTobeDeleted(Bson query) {
    // Fetch only the id
    final BasicDBObject keys = new BasicDBObject(Document.ID, 1);
    List<String> ids = new ArrayList<>();
    getNodeCollection().withReadPreference(store.getConfiguredReadPreference(NODES)).find(query).projection(keys).forEach((Block<BasicDBObject>) doc -> ids.add(getID(doc)));
    StringBuilder sb = new StringBuilder("Split documents with following ids were deleted as part of GC \n");
    Joiner.on(StandardSystemProperty.LINE_SEPARATOR.value()).appendTo(sb, ids);
    LOG.debug(sb.toString());
}
Also used : ReadPreference(com.mongodb.ReadPreference) MongoCollection(com.mongodb.client.MongoCollection) Iterables.transform(com.google.common.collect.Iterables.transform) LoggerFactory(org.slf4j.LoggerFactory) SD_MAX_REV_TIME_IN_SECS(org.apache.jackrabbit.oak.plugins.document.NodeDocument.SD_MAX_REV_TIME_IN_SECS) MODIFIED_IN_SECS(org.apache.jackrabbit.oak.plugins.document.NodeDocument.MODIFIED_IN_SECS) ArrayList(java.util.ArrayList) Collections.singletonList(java.util.Collections.singletonList) Filters(com.mongodb.client.model.Filters) Bson(org.bson.conversions.Bson) Utils(org.apache.jackrabbit.oak.plugins.document.util.Utils) Lists(com.google.common.collect.Lists) CloseableIterable(org.apache.jackrabbit.oak.plugins.document.util.CloseableIterable) StandardSystemProperty(com.google.common.base.StandardSystemProperty) SD_TYPE(org.apache.jackrabbit.oak.plugins.document.NodeDocument.SD_TYPE) NodeDocument(org.apache.jackrabbit.oak.plugins.document.NodeDocument) SplitDocType(org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType) RevisionVector(org.apache.jackrabbit.oak.plugins.document.RevisionVector) Nonnull(javax.annotation.Nonnull) PATH(org.apache.jackrabbit.oak.plugins.document.NodeDocument.PATH) Revision(org.apache.jackrabbit.oak.plugins.document.Revision) NODES(org.apache.jackrabbit.oak.plugins.document.Collection.NODES) DELETED_ONCE(org.apache.jackrabbit.oak.plugins.document.NodeDocument.DELETED_ONCE) Function(com.google.common.base.Function) SplitDocumentCleanUp(org.apache.jackrabbit.oak.plugins.document.SplitDocumentCleanUp) Logger(org.slf4j.Logger) NodeDocument.getModifiedInSecs(org.apache.jackrabbit.oak.plugins.document.NodeDocument.getModifiedInSecs) BasicDBObject(com.mongodb.BasicDBObject) Set(java.util.Set) VersionGCSupport(org.apache.jackrabbit.oak.plugins.document.VersionGCSupport) VersionGCStats(org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) Block(com.mongodb.Block) Clock(org.apache.jackrabbit.oak.stats.Clock) Predicate(com.google.common.base.Predicate) FindIterable(com.mongodb.client.FindIterable) Document(org.apache.jackrabbit.oak.plugins.document.Document) ID(org.apache.jackrabbit.oak.plugins.document.Document.ID) Pattern(java.util.regex.Pattern) Iterables.filter(com.google.common.collect.Iterables.filter) Joiner(com.google.common.base.Joiner) DEFAULT_NO_BRANCH(org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType.DEFAULT_NO_BRANCH) BasicDBObject(com.mongodb.BasicDBObject) ArrayList(java.util.ArrayList)

Example 15 with Block

use of com.mongodb.Block in project mongo-java-driver by mongodb.

the class ConnectionsSurvivePrimaryStepDownProseTest method setUp.

@Before
public void setUp() {
    assumeTrue(isDiscoverableReplicaSet() && serverVersionAtLeast(4, 0));
    connectionPoolListener = new TestConnectionPoolListener();
    MongoClientSettings settings = MongoClientSettings.builder(getMongoClientSettings()).retryWrites(false).applyToConnectionPoolSettings(new Block<ConnectionPoolSettings.Builder>() {

        @Override
        public void apply(final ConnectionPoolSettings.Builder builder) {
            builder.addConnectionPoolListener(connectionPoolListener);
        }
    }).build();
    collectionHelper = new CollectionHelper<Document>(new DocumentCodec(), new MongoNamespace(getDefaultDatabaseName(), COLLECTION_NAME));
    client = MongoClients.create(settings);
    MongoDatabase database = client.getDatabase(getDefaultDatabaseName());
    collection = client.getDatabase(getDefaultDatabaseName()).getCollection(COLLECTION_NAME);
    collection.withWriteConcern(WriteConcern.MAJORITY).drop();
    database.withWriteConcern(WriteConcern.MAJORITY).createCollection(COLLECTION_NAME);
}
Also used : TestConnectionPoolListener(com.mongodb.internal.connection.TestConnectionPoolListener) DocumentCodec(org.bson.codecs.DocumentCodec) Block(com.mongodb.Block) Fixture.getMongoClientSettings(com.mongodb.client.Fixture.getMongoClientSettings) MongoClientSettings(com.mongodb.MongoClientSettings) Document(org.bson.Document) MongoNamespace(com.mongodb.MongoNamespace) ConnectionPoolSettings(com.mongodb.connection.ConnectionPoolSettings) Before(org.junit.Before)

Aggregations

Block (com.mongodb.Block)18 Document (org.bson.Document)13 MongoClientSettings (com.mongodb.MongoClientSettings)7 MongoCollection (com.mongodb.client.MongoCollection)7 MongoDatabase (com.mongodb.client.MongoDatabase)6 ArrayList (java.util.ArrayList)6 HashMap (java.util.HashMap)6 Fixture.getMongoClientSettingsBuilder (com.mongodb.client.Fixture.getMongoClientSettingsBuilder)5 List (java.util.List)5 BsonDocument (org.bson.BsonDocument)5 ServerSettings (com.mongodb.connection.ServerSettings)4 MongoConnection (fr.inria.spirals.repairnator.serializer.mongodb.MongoConnection)4 Map (java.util.Map)4 Test (org.junit.Test)4 ClusterFixture.isServerlessTest (com.mongodb.ClusterFixture.isServerlessTest)3 ConnectionString (com.mongodb.ConnectionString)3 MongoException (com.mongodb.MongoException)3 SocketSettings (com.mongodb.connection.SocketSettings)3 HashSet (java.util.HashSet)3 ObjectId (org.bson.types.ObjectId)3