Search in sources :

Example 26 with ServerAddress

use of com.mongodb.ServerAddress in project drill by apache.

the class MongoGroupScan method getOperatorAffinity.

@Override
public List<EndpointAffinity> getOperatorAffinity() {
    watch.reset();
    watch.start();
    Map<String, DrillbitEndpoint> endpointMap = Maps.newHashMap();
    for (DrillbitEndpoint endpoint : storagePlugin.getContext().getBits()) {
        endpointMap.put(endpoint.getAddress(), endpoint);
        logger.debug("Endpoint address: {}", endpoint.getAddress());
    }
    Map<DrillbitEndpoint, EndpointAffinity> affinityMap = Maps.newHashMap();
    // multiple replicas for each chunk.
    for (Set<ServerAddress> addressList : chunksMapping.values()) {
        // meets affinity.
        for (ServerAddress address : addressList) {
            DrillbitEndpoint ep = endpointMap.get(address.getHost());
            if (ep != null) {
                EndpointAffinity affinity = affinityMap.get(ep);
                if (affinity == null) {
                    affinityMap.put(ep, new EndpointAffinity(ep, 1));
                } else {
                    affinity.addAffinity(1);
                }
                break;
            }
        }
    }
    logger.debug("Took {} µs to get operator affinity", watch.elapsed(TimeUnit.NANOSECONDS) / 1000);
    logger.debug("Affined drillbits : " + affinityMap.values());
    return Lists.newArrayList(affinityMap.values());
}
Also used : DrillbitEndpoint(org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint) ServerAddress(com.mongodb.ServerAddress) EndpointAffinity(org.apache.drill.exec.physical.EndpointAffinity)

Example 27 with ServerAddress

use of com.mongodb.ServerAddress in project drill by apache.

the class MongoGroupScan method getPreferredHosts.

@SuppressWarnings("unchecked")
private Set<ServerAddress> getPreferredHosts(MongoClient client, List<String> hosts) {
    Set<ServerAddress> addressList = Sets.newHashSet();
    MongoDatabase db = client.getDatabase(scanSpec.getDbName());
    ReadPreference readPreference = client.getReadPreference();
    Document command = db.runCommand(new Document("isMaster", 1));
    final String primaryHost = command.getString("primary");
    final List<String> hostsList = (List<String>) command.get("hosts");
    switch(readPreference.getName().toUpperCase()) {
        case "PRIMARY":
        case "PRIMARYPREFERRED":
            if (primaryHost == null) {
                return null;
            }
            addressList.add(new ServerAddress(primaryHost));
            return addressList;
        case "SECONDARY":
        case "SECONDARYPREFERRED":
            if (primaryHost == null || hostsList == null) {
                return null;
            }
            hostsList.remove(primaryHost);
            for (String host : hostsList) {
                addressList.add(new ServerAddress(host));
            }
            return addressList;
        case "NEAREST":
            if (hostsList == null) {
                return null;
            }
            for (String host : hostsList) {
                addressList.add(new ServerAddress(host));
            }
            return addressList;
        default:
            return null;
    }
}
Also used : ReadPreference(com.mongodb.ReadPreference) ServerAddress(com.mongodb.ServerAddress) List(java.util.List) ArrayList(java.util.ArrayList) Document(org.bson.Document) MongoDatabase(com.mongodb.client.MongoDatabase)

Example 28 with ServerAddress

use of com.mongodb.ServerAddress in project drill by apache.

the class TestMongoChunkAssignment method setUp.

@Before
public void setUp() throws UnknownHostException {
    chunksMapping = Maps.newHashMap();
    chunksInverseMapping = Maps.newLinkedHashMap();
    // entry1
    Set<ServerAddress> hosts_A = Sets.newHashSet();
    hosts_A.add(new ServerAddress(HOST_A));
    chunksMapping.put(dbName + "." + collectionName + "-01", hosts_A);
    chunksMapping.put(dbName + "." + collectionName + "-05", hosts_A);
    ChunkInfo chunk1Info = new ChunkInfo(Arrays.asList(HOST_A), dbName + "." + collectionName + "-01");
    chunk1Info.setMinFilters(Collections.<String, Object>emptyMap());
    Map<String, Object> chunk1MaxFilters = Maps.newHashMap();
    chunk1MaxFilters.put("name", Integer.valueOf(5));
    chunk1Info.setMaxFilters(chunk1MaxFilters);
    ChunkInfo chunk5Info = new ChunkInfo(Arrays.asList(HOST_A), dbName + "." + collectionName + "-05");
    Map<String, Object> chunk5MinFilters = Maps.newHashMap();
    chunk5MinFilters.put("name", Integer.valueOf(25));
    chunk5Info.setMinFilters(chunk5MinFilters);
    Map<String, Object> chunk5MaxFilters = Maps.newHashMap();
    chunk5MaxFilters.put("name", Integer.valueOf(30));
    chunk5Info.setMaxFilters(chunk5MaxFilters);
    List<ChunkInfo> chunkList = Arrays.asList(chunk1Info, chunk5Info);
    chunksInverseMapping.put(HOST_A, chunkList);
    // entry2
    Set<ServerAddress> hosts_B = Sets.newHashSet();
    hosts_A.add(new ServerAddress(HOST_B));
    chunksMapping.put(dbName + "." + collectionName + "-02", hosts_B);
    ChunkInfo chunk2Info = new ChunkInfo(Arrays.asList(HOST_B), dbName + "." + collectionName + "-02");
    Map<String, Object> chunk2MinFilters = Maps.newHashMap();
    chunk2MinFilters.put("name", Integer.valueOf(5));
    chunk2Info.setMinFilters(chunk2MinFilters);
    Map<String, Object> chunk2MaxFilters = Maps.newHashMap();
    chunk2MaxFilters.put("name", Integer.valueOf(15));
    chunk2Info.setMaxFilters(chunk2MaxFilters);
    chunkList = Arrays.asList(chunk2Info);
    chunksInverseMapping.put(HOST_B, chunkList);
    // enty3
    Set<ServerAddress> hosts_C = Sets.newHashSet();
    hosts_A.add(new ServerAddress(HOST_C));
    chunksMapping.put(dbName + "." + collectionName + "-03", hosts_C);
    chunksMapping.put(dbName + "." + collectionName + "-06", hosts_C);
    ChunkInfo chunk3Info = new ChunkInfo(Arrays.asList(HOST_C), dbName + "." + collectionName + "-03");
    Map<String, Object> chunk3MinFilters = Maps.newHashMap();
    chunk5MinFilters.put("name", Integer.valueOf(15));
    chunk3Info.setMinFilters(chunk3MinFilters);
    Map<String, Object> chunk3MaxFilters = Maps.newHashMap();
    chunk3MaxFilters.put("name", Integer.valueOf(20));
    chunk3Info.setMaxFilters(chunk3MaxFilters);
    ChunkInfo chunk6Info = new ChunkInfo(Arrays.asList(HOST_C), dbName + "." + collectionName + "-06");
    Map<String, Object> chunk6MinFilters = Maps.newHashMap();
    chunk5MinFilters.put("name", Integer.valueOf(25));
    chunk6Info.setMinFilters(chunk6MinFilters);
    Map<String, Object> chunk6MaxFilters = Maps.newHashMap();
    chunk5MaxFilters.put("name", Integer.valueOf(30));
    chunk6Info.setMaxFilters(chunk6MaxFilters);
    chunkList = Arrays.asList(chunk3Info, chunk6Info);
    chunksInverseMapping.put(HOST_C, chunkList);
    // entry4
    Set<ServerAddress> hosts_D = Sets.newHashSet();
    hosts_A.add(new ServerAddress(HOST_D));
    chunksMapping.put(dbName + "." + collectionName + "-04", hosts_D);
    ChunkInfo chunk4Info = new ChunkInfo(Arrays.asList(HOST_D), dbName + "." + collectionName + "-04");
    Map<String, Object> chunk4MinFilters = Maps.newHashMap();
    chunk4MinFilters.put("name", Integer.valueOf(20));
    chunk4Info.setMinFilters(chunk4MinFilters);
    Map<String, Object> chunk4MaxFilters = Maps.newHashMap();
    chunk4MaxFilters.put("name", Integer.valueOf(25));
    chunk4Info.setMaxFilters(chunk4MaxFilters);
    chunkList = Arrays.asList(chunk4Info);
    chunksInverseMapping.put(HOST_D, chunkList);
    mongoGroupScan = new MongoGroupScan();
    mongoGroupScan.setChunksMapping(chunksMapping);
    mongoGroupScan.setInverseChunsMapping(chunksInverseMapping);
    MongoScanSpec scanSpec = new MongoScanSpec(dbName, collectionName);
    mongoGroupScan.setScanSpec(scanSpec);
}
Also used : ChunkInfo(org.apache.drill.exec.store.mongo.common.ChunkInfo) ServerAddress(com.mongodb.ServerAddress) Before(org.junit.Before)

Example 29 with ServerAddress

use of com.mongodb.ServerAddress in project drill by apache.

the class MongoRecordReader method init.

private void init(MongoSubScan.MongoSubScanSpec subScanSpec) {
    List<String> hosts = subScanSpec.getHosts();
    List<ServerAddress> addresses = Lists.newArrayList();
    for (String host : hosts) {
        addresses.add(new ServerAddress(host));
    }
    MongoClient client = plugin.getClient(addresses);
    MongoDatabase db = client.getDatabase(subScanSpec.getDbName());
    this.unionEnabled = fragmentContext.getOptions().getOption(ExecConstants.ENABLE_UNION_TYPE);
    collection = db.getCollection(subScanSpec.getCollectionName(), BsonDocument.class);
}
Also used : MongoClient(com.mongodb.MongoClient) BsonDocument(org.bson.BsonDocument) ServerAddress(com.mongodb.ServerAddress) MongoDatabase(com.mongodb.client.MongoDatabase)

Example 30 with ServerAddress

use of com.mongodb.ServerAddress in project jmeter by apache.

the class MongoUtils method toServerAddresses.

public static List<ServerAddress> toServerAddresses(String connections) throws UnknownHostException {
    List<ServerAddress> addresses = new ArrayList<>();
    for (String connection : Arrays.asList(connections.split(","))) {
        int port = DEFAULT_PORT;
        String[] hostPort = connection.split(":");
        if (hostPort.length > 1 && !StringUtils.isEmpty(hostPort[1])) {
            port = Integer.parseInt(hostPort[1].trim());
        }
        addresses.add(new ServerAddress(hostPort[0], port));
    }
    return addresses;
}
Also used : ArrayList(java.util.ArrayList) ServerAddress(com.mongodb.ServerAddress)

Aggregations

ServerAddress (com.mongodb.ServerAddress)58 Test (org.junit.Test)22 MongoClient (com.mongodb.MongoClient)13 MongoCredential (com.mongodb.MongoCredential)9 ArrayList (java.util.ArrayList)9 Before (org.junit.Before)9 TagSet (com.mongodb.TagSet)4 ClusterDescription (com.mongodb.connection.ClusterDescription)4 ClusterSettings (com.mongodb.connection.ClusterSettings)4 MongoClient (com.mongodb.reactivestreams.client.MongoClient)4 Tag (com.mongodb.Tag)3 MongoDatabase (com.mongodb.client.MongoDatabase)3 ServerDescription (com.mongodb.connection.ServerDescription)3 UnknownHostException (java.net.UnknownHostException)3 Date (java.util.Date)3 ChunkInfo (org.apache.drill.exec.store.mongo.common.ChunkInfo)3 BsonDocument (org.bson.BsonDocument)3 DataXException (com.alibaba.datax.common.exception.DataXException)2 ConnectionString (com.mongodb.ConnectionString)2 MongoClientURI (com.mongodb.MongoClientURI)2