Examples with BasicDBObject com.mongodb.BasicDBObject used on opensource projects

Example 21 with BasicDBObject

use of com.mongodb.BasicDBObject in project mongomvcc by igd-geo.

the class MongoDBVMaintenance method pruneDanglingCommits.

@Override
public long pruneDanglingCommits(long expiry, TimeUnit unit) {
    long[] cids = findDanglingCommits(expiry, unit);
    DBCollection collCommits = _db.getDB().getCollection(MongoDBConstants.COLLECTION_COMMITS);
    //delete commits in chunks, so we avoid sending an array that is
    //larger than the maximum document size
    final int sliceCount = 1000;
    for (int i = 0; i < cids.length; i += sliceCount) {
        int maxSliceCount = Math.min(sliceCount, cids.length - i);
        long[] slice = new long[maxSliceCount];
        System.arraycopy(cids, i, slice, 0, maxSliceCount);
        collCommits.remove(new BasicDBObject(MongoDBConstants.ID, new BasicDBObject("$in", slice)));
    }
    return cids.length;
}

Example 22 with BasicDBObject

use of com.mongodb.BasicDBObject in project mongo-hadoop by mongodb.

the class StandaloneMongoSplitter method calculateSplits.

@Override
public List<InputSplit> calculateSplits() throws SplitFailedException {
    final DBObject splitKey = MongoConfigUtil.getInputSplitKey(getConfiguration());
    final DBObject splitKeyMax = MongoConfigUtil.getMaxSplitKey(getConfiguration());
    final DBObject splitKeyMin = MongoConfigUtil.getMinSplitKey(getConfiguration());
    final int splitSize = MongoConfigUtil.getSplitSize(getConfiguration());
    final MongoClientURI inputURI;
    DBCollection inputCollection = null;
    final ArrayList<InputSplit> returnVal;
    try {
        inputURI = MongoConfigUtil.getInputURI(getConfiguration());
        MongoClientURI authURI = MongoConfigUtil.getAuthURI(getConfiguration());
        if (authURI != null) {
            inputCollection = MongoConfigUtil.getCollectionWithAuth(inputURI, authURI);
        } else {
            inputCollection = MongoConfigUtil.getCollection(inputURI);
        }
        returnVal = new ArrayList<InputSplit>();
        final String ns = inputCollection.getFullName();
        if (LOG.isDebugEnabled()) {
            LOG.debug(String.format("Running splitVector on namespace: %s.%s; hosts: %s", inputURI.getDatabase(), inputURI.getCollection(), inputURI.getHosts()));
        }
        final DBObject cmd = BasicDBObjectBuilder.start("splitVector", ns).add("keyPattern", splitKey).add("min", splitKeyMin).add("max", splitKeyMax).add("force", false).add("maxChunkSize", splitSize).get();
        CommandResult data;
        boolean ok = true;
        try {
            data = inputCollection.getDB().getSisterDB(inputURI.getDatabase()).command(cmd, ReadPreference.primary());
        } catch (final MongoException e) {
            // 2.0 servers throw exceptions rather than info in a CommandResult
            data = null;
            LOG.info(e.getMessage(), e);
            if (e.getMessage().contains("unrecognized command: splitVector")) {
                ok = false;
            } else {
                throw e;
            }
        }
        if (data != null) {
            if (data.containsField("$err")) {
                throw new SplitFailedException("Error calculating splits: " + data);
            } else if (!data.get("ok").equals(1.0)) {
                ok = false;
            }
        }
        if (!ok) {
            final CommandResult stats = inputCollection.getStats();
            if (stats.containsField("primary")) {
                final DBCursor shards = inputCollection.getDB().getSisterDB("config").getCollection("shards").find(new BasicDBObject("_id", stats.getString("primary")));
                try {
                    if (shards.hasNext()) {
                        final DBObject shard = shards.next();
                        final String host = ((String) shard.get("host")).replace(shard.get("_id") + "/", "");
                        final MongoClientURI shardHost;
                        if (authURI != null) {
                            shardHost = new MongoClientURIBuilder(authURI).host(host).build();
                        } else {
                            shardHost = new MongoClientURIBuilder(inputURI).host(host).build();
                        }
                        MongoClient shardClient = null;
                        try {
                            shardClient = new MongoClient(shardHost);
                            data = shardClient.getDB(shardHost.getDatabase()).command(cmd, ReadPreference.primary());
                        } catch (final Exception e) {
                            LOG.error(e.getMessage(), e);
                        } finally {
                            if (shardClient != null) {
                                shardClient.close();
                            }
                        }
                    }
                } finally {
                    shards.close();
                }
            }
            if (data != null && !data.get("ok").equals(1.0)) {
                throw new SplitFailedException("Unable to calculate input splits: " + data.get("errmsg"));
            }
        }
        // Comes in a format where "min" and "max" are implicit
        // and each entry is just a boundary key; not ranged
        final BasicDBList splitData = (BasicDBList) data.get("splitKeys");
        if (splitData.size() == 0) {
            LOG.warn("WARNING: No Input Splits were calculated by the split code. Proceeding with a *single* split. Data may be too" + " small, try lowering 'mongo.input.split_size' if this is undesirable.");
        }
        // Lower boundary of the first min split
        BasicDBObject lastKey = null;
        // If splitKeyMin was given, use it as first boundary.
        if (!splitKeyMin.toMap().isEmpty()) {
            lastKey = new BasicDBObject(splitKeyMin.toMap());
        }
        for (final Object aSplitData : splitData) {
            final BasicDBObject currentKey = (BasicDBObject) aSplitData;
            returnVal.add(createSplitFromBounds(lastKey, currentKey));
            lastKey = currentKey;
        }
        BasicDBObject maxKey = null;
        // If splitKeyMax was given, use it as last boundary.
        if (!splitKeyMax.toMap().isEmpty()) {
            maxKey = new BasicDBObject(splitKeyMax.toMap());
        }
        // Last max split
        final MongoInputSplit lastSplit = createSplitFromBounds(lastKey, maxKey);
        returnVal.add(lastSplit);
    } finally {
        if (inputCollection != null) {
            MongoConfigUtil.close(inputCollection.getDB().getMongo());
        }
    }
    if (MongoConfigUtil.isFilterEmptySplitsEnabled(getConfiguration())) {
        return filterEmptySplits(returnVal);
    }
    return returnVal;
}

Example 23 with BasicDBObject

use of com.mongodb.BasicDBObject in project mongo-hadoop by mongodb.

the class MongoConfigUtil method getDBObject.

public static DBObject getDBObject(final Configuration conf, final String key) {
    try {
        final String json = conf.get(key);
        final DBObject obj = (DBObject) JSON.parse(json);
        if (obj == null) {
            return new BasicDBObject();
        } else {
            return obj;
        }
    } catch (final Exception e) {
        throw new IllegalArgumentException("Provided JSON String is not representable/parseable as a DBObject.", e);
    }
}

Example 24 with BasicDBObject

use of com.mongodb.BasicDBObject in project mongo-hadoop by mongodb.

the class MongoConfigUtil method getInputSplitKey.

public static DBObject getInputSplitKey(final Configuration conf) {
    try {
        final String json = getInputSplitKeyPattern(conf);
        final DBObject obj = (DBObject) JSON.parse(json);
        if (obj == null) {
            return new BasicDBObject("_id", 1);
        } else {
            return obj;
        }
    } catch (final Exception e) {
        throw new IllegalArgumentException("Provided JSON String is not representable/parsable as a DBObject.", e);
    }
}

Example 25 with BasicDBObject

use of com.mongodb.BasicDBObject in project mongo-hadoop by mongodb.

the class GridFSInputFormatTest method getConfiguration.

private static Configuration getConfiguration() {
    Configuration conf = new Configuration();
    MongoConfigUtil.setInputURI(conf, "mongodb://localhost:27017/mongo_hadoop.fs");
    MongoConfigUtil.setQuery(conf, new BasicDBObject("filename", "README.md"));
    return conf;
}