Examples with DrillbitEndpoint org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint used on opensource projects

Example 1 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class MongoGroupScan method getOperatorAffinity.

@Override
public List<EndpointAffinity> getOperatorAffinity() {
    watch.reset();
    watch.start();
    Map<String, DrillbitEndpoint> endpointMap = Maps.newHashMap();
    for (DrillbitEndpoint endpoint : storagePlugin.getContext().getBits()) {
        endpointMap.put(endpoint.getAddress(), endpoint);
        logger.debug("Endpoint address: {}", endpoint.getAddress());
    }
    Map<DrillbitEndpoint, EndpointAffinity> affinityMap = Maps.newHashMap();
    // multiple replicas for each chunk.
    for (Set<ServerAddress> addressList : chunksMapping.values()) {
        // meets affinity.
        for (ServerAddress address : addressList) {
            DrillbitEndpoint ep = endpointMap.get(address.getHost());
            if (ep != null) {
                EndpointAffinity affinity = affinityMap.get(ep);
                if (affinity == null) {
                    affinityMap.put(ep, new EndpointAffinity(ep, 1));
                } else {
                    affinity.addAffinity(1);
                }
                break;
            }
        }
    }
    logger.debug("Took {} µs to get operator affinity", watch.elapsed(TimeUnit.NANOSECONDS) / 1000);
    logger.debug("Affined drillbits : " + affinityMap.values());
    return Lists.newArrayList(affinityMap.values());
}

Example 2 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class DrillClient method parseAndVerifyEndpoints.

/**
   * Populates the endpointlist with drillbits information provided in the connection string by client.
   * For direct connection we can have connection string with drillbit property as below:
   * <dl>
   *   <dt>drillbit=ip</dt>
   *   <dd>use the ip specified as the Foreman ip with default port in config file</dd>
   *   <dt>drillbit=ip:port</dt>
   *   <dd>use the ip and port specified as the Foreman ip and port</dd>
   *   <dt>drillbit=ip1:port1,ip2:port2,...</dt>
   *   <dd>randomly select the ip and port pair from the specified list as the Foreman ip and port.</dd>
   * </dl>
   *
   * @param drillbits string with drillbit value provided in connection string
   * @param defaultUserPort string with default userport of drillbit specified in config file
   * @return list of drillbit endpoints parsed from connection string
   * @throws InvalidConnectionInfoException if the connection string has invalid or no drillbit information
   */
static List<DrillbitEndpoint> parseAndVerifyEndpoints(String drillbits, String defaultUserPort) throws InvalidConnectionInfoException {
    // If no drillbits is provided then throw exception
    drillbits = drillbits.trim();
    if (drillbits.isEmpty()) {
        throw new InvalidConnectionInfoException("No drillbit information specified in the connection string");
    }
    final List<DrillbitEndpoint> endpointList = new ArrayList<>();
    final String[] connectInfo = drillbits.split(",");
    // Fetch ip address and port information for each drillbit and populate the list
    for (String drillbit : connectInfo) {
        // Trim all the empty spaces and check if the entry is empty string.
        // Ignore the empty ones.
        drillbit = drillbit.trim();
        if (!drillbit.isEmpty()) {
            // Verify if we have only ":" or only ":port" pattern
            if (drillbit.charAt(0) == ':') {
                // Invalid drillbit information
                throw new InvalidConnectionInfoException("Malformed connection string with drillbit hostname or " + "hostaddress missing for an entry: " + drillbit);
            }
            // We are now sure that each ip:port entry will have both the values atleast once.
            // Split each drillbit connection string to get ip address and port value
            final String[] drillbitInfo = drillbit.split(":");
            // Check if we have more than one port
            if (drillbitInfo.length > 2) {
                throw new InvalidConnectionInfoException("Malformed connection string with more than one port in a " + "drillbit entry: " + drillbit);
            }
            // At this point we are sure that drillbitInfo has atleast hostname or host address
            // trim all the empty spaces which might be present in front of hostname or
            // host address information
            final String ipAddress = drillbitInfo[0].trim();
            String port = defaultUserPort;
            if (drillbitInfo.length == 2) {
                // We have a port value also given by user. trim all the empty spaces between : and port value before
                // validating the correctness of value.
                port = drillbitInfo[1].trim();
            }
            try {
                final DrillbitEndpoint endpoint = DrillbitEndpoint.newBuilder().setAddress(ipAddress).setUserPort(Integer.parseInt(port)).build();
                endpointList.add(endpoint);
            } catch (NumberFormatException e) {
                throw new InvalidConnectionInfoException("Malformed port value in entry: " + ipAddress + ":" + port + " " + "passed in connection string");
            }
        }
    }
    if (endpointList.size() == 0) {
        throw new InvalidConnectionInfoException("No valid drillbit information specified in the connection string");
    }
    return endpointList;
}

Example 3 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class DrillClient method connect.

public synchronized void connect(String connect, Properties props) throws RpcException {
    if (connected) {
        return;
    }
    properties = DrillProperties.createFromProperties(props);
    final List<DrillbitEndpoint> endpoints = new ArrayList<>();
    if (isDirectConnection) {
        // Populate the endpoints list with all the drillbit information provided in the connection string
        endpoints.addAll(parseAndVerifyEndpoints(properties.getProperty(DrillProperties.DRILLBIT_CONNECTION), config.getString(ExecConstants.INITIAL_USER_PORT)));
    } else {
        if (ownsZkConnection) {
            try {
                this.clusterCoordinator = new ZKClusterCoordinator(this.config, connect);
                this.clusterCoordinator.start(10000);
            } catch (Exception e) {
                throw new RpcException("Failure setting up ZK for client.", e);
            }
        }
        endpoints.addAll(clusterCoordinator.getAvailableEndpoints());
        // Make sure we have at least one endpoint in the list
        checkState(!endpoints.isEmpty(), "No active Drillbit endpoint found from ZooKeeper. Check connection parameters?");
    }
    // shuffle the collection then get the first endpoint
    Collections.shuffle(endpoints);
    eventLoopGroup = createEventLoop(config.getInt(ExecConstants.CLIENT_RPC_THREADS), "Client-");
    executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), new NamedThreadFactory("drill-client-executor-")) {

        @Override
        protected void afterExecute(final Runnable r, final Throwable t) {
            if (t != null) {
                logger.error("{}.run() leaked an exception.", r.getClass().getName(), t);
            }
            super.afterExecute(r, t);
        }
    };
    final String connectTriesConf = properties.getProperty(DrillProperties.TRIES, "5");
    int connectTriesVal;
    try {
        connectTriesVal = Math.min(endpoints.size(), Integer.parseInt(connectTriesConf));
    } catch (NumberFormatException e) {
        throw new InvalidConnectionInfoException("Invalid tries value: " + connectTriesConf + " specified in " + "connection string");
    }
    // If the value provided in the connection string is <=0 then override with 1 since we want to try connecting
    // at least once
    connectTriesVal = Math.max(1, connectTriesVal);
    int triedEndpointIndex = 0;
    DrillbitEndpoint endpoint;
    while (triedEndpointIndex < connectTriesVal) {
        client = new UserClient(clientName, config, supportComplexTypes, allocator, eventLoopGroup, executor);
        endpoint = endpoints.get(triedEndpointIndex);
        logger.debug("Connecting to server {}:{}", endpoint.getAddress(), endpoint.getUserPort());
        if (!properties.containsKey(DrillProperties.SERVICE_HOST)) {
            properties.setProperty(DrillProperties.SERVICE_HOST, endpoint.getAddress());
        }
        try {
            connect(endpoint);
            connected = true;
            logger.info("Successfully connected to server {}:{}", endpoint.getAddress(), endpoint.getUserPort());
            break;
        } catch (NonTransientRpcException ex) {
            logger.error("Connection to {}:{} failed with error {}. Not retrying anymore", endpoint.getAddress(), endpoint.getUserPort(), ex.getMessage());
            throw ex;
        } catch (RpcException ex) {
            ++triedEndpointIndex;
            logger.error("Attempt {}: Failed to connect to server {}:{}", triedEndpointIndex, endpoint.getAddress(), endpoint.getUserPort());
            // Throw exception when we have exhausted all the tries without having a successful connection
            if (triedEndpointIndex == connectTriesVal) {
                throw ex;
            }
            // Close the connection here to avoid calling close twice in case when all tries are exhausted.
            // Since DrillClient.close is also calling client.close
            client.close();
        }
    }
}

Example 4 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class TestMongoChunkAssignment method testMongoGroupScanAssignmentMix.

@Test
public void testMongoGroupScanAssignmentMix() throws UnknownHostException, ExecutionSetupException {
    final List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    final DrillbitEndpoint DB_A = DrillbitEndpoint.newBuilder().setAddress(HOST_A).setControlPort(1234).build();
    endpoints.add(DB_A);
    endpoints.add(DB_A);
    final DrillbitEndpoint DB_B = DrillbitEndpoint.newBuilder().setAddress(HOST_B).setControlPort(1234).build();
    endpoints.add(DB_B);
    final DrillbitEndpoint DB_D = DrillbitEndpoint.newBuilder().setAddress(HOST_D).setControlPort(1234).build();
    endpoints.add(DB_D);
    final DrillbitEndpoint DB_X = DrillbitEndpoint.newBuilder().setAddress(HOST_X).setControlPort(1234).build();
    endpoints.add(DB_X);
    mongoGroupScan.applyAssignments(endpoints);
    // assignments for chunks on host A, assign on drill bit A
    assertEquals(1, mongoGroupScan.getSpecificScan(0).getChunkScanSpecList().size());
    // assignments for chunks on host A, assign on drill bit A
    assertEquals(1, mongoGroupScan.getSpecificScan(1).getChunkScanSpecList().size());
    // assignments for chunks on host B, assign on drill bit B
    assertEquals(1, mongoGroupScan.getSpecificScan(2).getChunkScanSpecList().size());
    // assignments for chunks on host D, assign on drill bit D
    assertEquals(1, mongoGroupScan.getSpecificScan(3).getChunkScanSpecList().size());
    // assignments for chunks on host C, assign on drill bit X
    assertEquals(2, mongoGroupScan.getSpecificScan(4).getChunkScanSpecList().size());
}

Example 5 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class TestMongoChunkAssignment method testMongoGroupScanAssignmentAllAffinity.

@Test
public void testMongoGroupScanAssignmentAllAffinity() throws UnknownHostException, ExecutionSetupException {
    final List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    final DrillbitEndpoint DB_A = DrillbitEndpoint.newBuilder().setAddress(HOST_A).setControlPort(1234).build();
    endpoints.add(DB_A);
    final DrillbitEndpoint DB_B = DrillbitEndpoint.newBuilder().setAddress(HOST_B).setControlPort(1234).build();
    endpoints.add(DB_B);
    final DrillbitEndpoint DB_C = DrillbitEndpoint.newBuilder().setAddress(HOST_C).setControlPort(1234).build();
    endpoints.add(DB_C);
    final DrillbitEndpoint DB_D = DrillbitEndpoint.newBuilder().setAddress(HOST_D).setControlPort(1234).build();
    endpoints.add(DB_D);
    mongoGroupScan.applyAssignments(endpoints);
    // assignments for chunks on host A, assign on drill bit A
    assertEquals(2, mongoGroupScan.getSpecificScan(0).getChunkScanSpecList().size());
    // assignments for chunks on host B, assign on drill bit B
    assertEquals(1, mongoGroupScan.getSpecificScan(1).getChunkScanSpecList().size());
    // assignments for chunks on host C, assign on drill bit C
    assertEquals(2, mongoGroupScan.getSpecificScan(2).getChunkScanSpecList().size());
    // assignments for chunks on host D, assign on drill bit D
    assertEquals(1, mongoGroupScan.getSpecificScan(3).getChunkScanSpecList().size());
}