Examples with MongoClientURI com.mongodb.MongoClientURI used on opensource projects

Example 51 with MongoClientURI

use of com.mongodb.MongoClientURI in project mongo-hadoop by mongodb.

the class MongoInputSplit method readFields.

@Override
public void readFields(final DataInput in) throws IOException {
    BSONCallback cb = new BasicBSONCallback();
    BSONObject spec;
    byte[] l = new byte[4];
    in.readFully(l);
    int dataLen = org.bson.io.Bits.readInt(l);
    byte[] data = new byte[dataLen + 4];
    System.arraycopy(l, 0, data, 0, 4);
    in.readFully(data, 4, dataLen - 4);
    _bsonDecoder.decode(data, cb);
    spec = (BSONObject) cb.get();
    setInputURI(new MongoClientURI((String) spec.get("inputURI")));
    if (spec.get("authURI") != null) {
        setAuthURI(new MongoClientURI((String) spec.get("authURI")));
    } else {
        setAuthURI((MongoClientURI) null);
    }
    setKeyField((String) spec.get("keyField"));
    BSONObject temp = (BSONObject) spec.get("fields");
    setFields(temp != null ? new BasicDBObject(temp.toMap()) : null);
    temp = (BSONObject) spec.get("query");
    setQuery(temp != null ? new BasicDBObject(temp.toMap()) : null);
    temp = (BSONObject) spec.get("sort");
    setSort(temp != null ? new BasicDBObject(temp.toMap()) : null);
    temp = (BSONObject) spec.get("min");
    setMin(temp != null ? new BasicDBObject(temp.toMap()) : null);
    temp = (BSONObject) spec.get("max");
    setMax(temp != null ? new BasicDBObject(temp.toMap()) : null);
    setLimit((Integer) spec.get("limit"));
    setSkip((Integer) spec.get("skip"));
    setNoTimeout((Boolean) spec.get("notimeout"));
}

Example 52 with MongoClientURI

use of com.mongodb.MongoClientURI in project mongo-hadoop by mongodb.

the class MongoCollectionSplitter method getConfigDB.

protected DB getConfigDB() {
    Mongo mongo;
    MongoClientURI inputURI = MongoConfigUtil.getInputURI(getConfiguration());
    MongoClientURI authURI = MongoConfigUtil.getAuthURI(getConfiguration());
    final DBCollection inputCollection;
    if (authURI != null && authURI.getUsername() != null && authURI.getPassword() != null) {
        inputCollection = MongoConfigUtil.getCollectionWithAuth(inputURI, authURI);
    } else {
        inputCollection = MongoConfigUtil.getCollection(inputURI);
    }
    DB db = inputCollection.getDB();
    mongo = db.getMongo();
    if (authURI != null) {
        if (authURI.getUsername() != null && authURI.getPassword() != null) {
            authDB = mongo.getDB(authURI.getDatabase());
        }
    }
    return mongo.getDB("config");
}

Example 53 with MongoClientURI

use of com.mongodb.MongoClientURI in project mongo-hadoop by mongodb.

the class ShardChunkMongoSplitter method calculateSplitsFromChunks.

/**
     * Get a list of InputSplits based on a list of MongoDB shard chunks, the shard key, and a
     * mapping of shard names to host names. This is used internally by {@link #calculateSplits()}.
     *
     * @param chunks Chunk documents from the config.chunks collection.
     * @param shardsMap A map of shard name -> an array of hostnames.
     * @return A list of InputSplits.
     */
List<InputSplit> calculateSplitsFromChunks(final List<DBObject> chunks, final Map<String, List<String>> shardsMap) throws SplitFailedException {
    boolean targetShards = MongoConfigUtil.canReadSplitsFromShards(getConfiguration());
    List<String> mongosHostNames = MongoConfigUtil.getInputMongosHosts(getConfiguration());
    MongoClientURI inputURI = MongoConfigUtil.getInputURI(getConfiguration());
    if (targetShards && mongosHostNames.size() > 0) {
        throw new SplitFailedException("Setting both mongo.input.split.read_from_shards and mongo.input.mongos_hosts" + " does not make sense. ");
    }
    Map<String, String> mongosMap = null;
    if (mongosHostNames.size() > 0) {
        // Build a map of host -> mongos host string (incl. port)
        mongosMap = new HashMap<String, String>();
        for (String mongosHostName : mongosHostNames) {
            String[] hostAndPort = mongosHostName.split(":");
            mongosMap.put(hostAndPort[0], mongosHostName);
        }
    }
    List<InputSplit> splits = new ArrayList<InputSplit>(chunks.size());
    for (DBObject chunk : chunks) {
        BasicDBObject chunkLowerBound = (BasicDBObject) chunk.get("min");
        BasicDBObject chunkUpperBound = (BasicDBObject) chunk.get("max");
        MongoInputSplit chunkSplit = createSplitFromBounds(chunkLowerBound, chunkUpperBound);
        chunkSplit.setInputURI(inputURI);
        String shard = (String) chunk.get("shard");
        if (targetShards) {
            //The job is configured to target shards, so replace the
            //mongos hostname with the host of the shard's servers
            List<String> shardHosts = shardsMap.get(shard);
            if (shardHosts == null) {
                throw new SplitFailedException("Couldn't find shard ID: " + shard + " in config.shards.");
            }
            MongoClientURI newURI = rewriteURI(inputURI, shardHosts);
            chunkSplit.setInputURI(newURI);
        } else if (mongosMap != null) {
            // Try to use a mongos collocated with one of the shard hosts for the input
            // split. If the user has their Hadoop/MongoDB clusters configured correctly,
            // this will allow for reading without having to transfer data over a network.
            // Note that MongoInputSplit.getLocations() just returns the hostnames from its
            // input URI.
            List<String> chunkHosts = shardsMap.get(shard);
            String mongosHost = null;
            for (String chunkHost : chunkHosts) {
                String[] hostAndPort = chunkHost.split(":");
                mongosHost = mongosMap.get(hostAndPort[0]);
                if (mongosHost != null) {
                    break;
                }
            }
            if (null == mongosHost) {
                // Fall back just to using the given input URI.
                chunkSplit.setInputURI(inputURI);
            } else {
                LOG.info("Will read split " + chunkSplit + " from mongos " + mongosHost);
                chunkSplit.setInputURI(rewriteURI(inputURI, mongosHost));
            }
        }
        // Add this split to the list for the current shard.
        chunkSplit.setKeyField(MongoConfigUtil.getInputKey(getConfiguration()));
        splits.add(chunkSplit);
    }
    if (MongoConfigUtil.isFilterEmptySplitsEnabled(getConfiguration())) {
        return filterEmptySplits(splits);
    }
    return splits;
}

Example 54 with MongoClientURI

use of com.mongodb.MongoClientURI in project mongo-hadoop by mongodb.

the class MongoConfigUtil method close.

public static void close(final Mongo client) {
    MongoClientURI uri = URI_MAP.get().remove(client);
    if (uri != null) {
        MongoClient remove;
        remove = CLIENTS.get().remove(uri);
        if (remove != client) {
            throw new IllegalStateException("different clients found");
        }
        client.close();
    }
}

Example 55 with MongoClientURI

use of com.mongodb.MongoClientURI in project drill by apache.

the class MongoPersistentStoreProvider method start.

@Override
public void start() throws IOException {
    MongoClientURI clientURI = new MongoClientURI(mongoURL);
    client = new MongoClient(clientURI);
    MongoDatabase db = client.getDatabase(clientURI.getDatabase());
    collection = db.getCollection(clientURI.getCollection()).withWriteConcern(WriteConcern.JOURNALED);
    Bson index = Indexes.ascending(pKey);
    collection.createIndex(index);
}