Examples with HostAndPort org.apache.accumulo.core.util.HostAndPort used on opensource projects

Example 16 with HostAndPort

use of org.apache.accumulo.core.util.HostAndPort in project accumulo by apache.

the class LocalityCheck method addBlocks.

private void addBlocks(VolumeManager fs, String host, ArrayList<String> files, Map<String, Long> totalBlocks, Map<String, Long> localBlocks) throws Exception {
    long allBlocks = 0;
    long matchingBlocks = 0;
    if (!totalBlocks.containsKey(host)) {
        totalBlocks.put(host, 0L);
        localBlocks.put(host, 0L);
    }
    for (String file : files) {
        Path filePath = new Path(file);
        FileSystem ns = fs.getVolumeByPath(filePath).getFileSystem();
        FileStatus fileStatus = ns.getFileStatus(filePath);
        BlockLocation[] fileBlockLocations = ns.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
        for (BlockLocation blockLocation : fileBlockLocations) {
            allBlocks++;
            for (String location : blockLocation.getHosts()) {
                HostAndPort hap = HostAndPort.fromParts(location, 0);
                if (hap.getHost().equals(host)) {
                    matchingBlocks++;
                    break;
                }
            }
        }
    }
    totalBlocks.put(host, allBlocks + totalBlocks.get(host));
    localBlocks.put(host, matchingBlocks + localBlocks.get(host));
}

Example 17 with HostAndPort

use of org.apache.accumulo.core.util.HostAndPort in project accumulo by apache.

the class TServerUtils method startTServer.

/**
 * Start the appropriate Thrift server (SSL or non-blocking server) for the given parameters. Non-null SSL parameters will cause an SSL server to be started.
 *
 * @return A ServerAddress encapsulating the Thrift server created and the host/port which it is bound to.
 */
public static ServerAddress startTServer(ThriftServerType serverType, TimedProcessor processor, TProtocolFactory protocolFactory, String serverName, String threadName, int numThreads, int numSTThreads, long timeBetweenThreadChecks, long maxMessageSize, SslConnectionParams sslParams, SaslServerConnectionParams saslParams, long serverSocketTimeout, HostAndPort... addresses) throws TTransportException {
    // This is presently not supported. It's hypothetically possible, I believe, to work, but it would require changes in how the transports
    // work at the Thrift layer to ensure that both the SSL and SASL handshakes function. SASL's quality of protection addresses privacy issues.
    checkArgument(!(sslParams != null && saslParams != null), "Cannot start a Thrift server using both SSL and SASL");
    ServerAddress serverAddress = null;
    for (HostAndPort address : addresses) {
        try {
            switch(serverType) {
                case SSL:
                    log.debug("Instantiating SSL Thrift server");
                    serverAddress = createSslThreadPoolServer(address, processor, protocolFactory, serverSocketTimeout, sslParams, serverName, numThreads, numSTThreads, timeBetweenThreadChecks);
                    break;
                case SASL:
                    log.debug("Instantiating SASL Thrift server");
                    serverAddress = createSaslThreadPoolServer(address, processor, protocolFactory, serverSocketTimeout, saslParams, serverName, threadName, numThreads, numSTThreads, timeBetweenThreadChecks);
                    break;
                case THREADPOOL:
                    log.debug("Instantiating unsecure TThreadPool Thrift server");
                    serverAddress = createBlockingServer(address, processor, protocolFactory, maxMessageSize, serverName, numThreads, numSTThreads, timeBetweenThreadChecks);
                    break;
                // Intentional passthrough -- Our custom wrapper around HsHa is the default
                case CUSTOM_HS_HA:
                default:
                    log.debug("Instantiating default, unsecure custom half-async Thrift server");
                    serverAddress = createNonBlockingServer(address, processor, protocolFactory, serverName, threadName, numThreads, numSTThreads, timeBetweenThreadChecks, maxMessageSize);
            }
            break;
        } catch (TTransportException e) {
            log.warn("Error attempting to create server at {}. Error: {}", address.toString(), e.getMessage());
        }
    }
    if (null == serverAddress) {
        throw new TTransportException("Unable to create server on addresses: " + Arrays.toString(addresses));
    }
    final TServer finalServer = serverAddress.server;
    Runnable serveTask = new Runnable() {

        @Override
        public void run() {
            try {
                finalServer.serve();
            } catch (Error e) {
                Halt.halt("Unexpected error in TThreadPoolServer " + e + ", halting.", 1);
            }
        }
    };
    serveTask = new LoggingRunnable(TServerUtils.log, serveTask);
    Thread thread = new Daemon(serveTask, threadName);
    thread.start();
    // check for the special "bind to everything address"
    if (serverAddress.address.getHost().equals("0.0.0.0")) {
        // can't get the address from the bind, so we'll do our best to invent our hostname
        try {
            serverAddress = new ServerAddress(finalServer, HostAndPort.fromParts(InetAddress.getLocalHost().getHostName(), serverAddress.address.getPort()));
        } catch (UnknownHostException e) {
            throw new TTransportException(e);
        }
    }
    return serverAddress;
}

Example 18 with HostAndPort

use of org.apache.accumulo.core.util.HostAndPort in project accumulo by apache.

the class TServerUtils method startServer.

/**
 * Start a server, at the given port, or higher, if that port is not available.
 *
 * @param service
 *          RPC configuration
 * @param portHintProperty
 *          the port to attempt to open, can be zero, meaning "any available port"
 * @param processor
 *          the service to be started
 * @param serverName
 *          the name of the class that is providing the service
 * @param threadName
 *          name this service's thread for better debugging
 * @param portSearchProperty
 *          A boolean Property to control if port-search should be used, or null to disable
 * @param minThreadProperty
 *          A Property to control the minimum number of threads in the pool
 * @param timeBetweenThreadChecksProperty
 *          A Property to control the amount of time between checks to resize the thread pool
 * @param maxMessageSizeProperty
 *          A Property to control the maximum Thrift message size accepted
 * @return the server object created, and the port actually used
 * @throws UnknownHostException
 *           when we don't know our own address
 */
public static ServerAddress startServer(AccumuloServerContext service, String hostname, Property portHintProperty, TProcessor processor, String serverName, String threadName, Property portSearchProperty, Property minThreadProperty, Property timeBetweenThreadChecksProperty, Property maxMessageSizeProperty) throws UnknownHostException {
    final AccumuloConfiguration config = service.getConfiguration();
    final int[] portHint = config.getPort(portHintProperty);
    int minThreads = 2;
    if (minThreadProperty != null)
        minThreads = config.getCount(minThreadProperty);
    long timeBetweenThreadChecks = 1000;
    if (timeBetweenThreadChecksProperty != null)
        timeBetweenThreadChecks = config.getTimeInMillis(timeBetweenThreadChecksProperty);
    long maxMessageSize = 10 * 1000 * 1000;
    if (maxMessageSizeProperty != null)
        maxMessageSize = config.getAsBytes(maxMessageSizeProperty);
    boolean portSearch = false;
    if (portSearchProperty != null)
        portSearch = config.getBoolean(portSearchProperty);
    final int simpleTimerThreadpoolSize = config.getCount(Property.GENERAL_SIMPLETIMER_THREADPOOL_SIZE);
    final ThriftServerType serverType = service.getThriftServerType();
    if (ThriftServerType.SASL == serverType) {
        processor = updateSaslProcessor(serverType, processor);
    }
    // create the TimedProcessor outside the port search loop so we don't try to register the same metrics mbean more than once
    TimedProcessor timedProcessor = new TimedProcessor(config, processor, serverName, threadName);
    HostAndPort[] addresses = getHostAndPorts(hostname, portHint);
    try {
        return TServerUtils.startTServer(serverType, timedProcessor, serverName, threadName, minThreads, simpleTimerThreadpoolSize, timeBetweenThreadChecks, maxMessageSize, service.getServerSslParams(), service.getSaslParams(), service.getClientTimeoutInMillis(), addresses);
    } catch (TTransportException e) {
        if (portSearch) {
            HostAndPort last = addresses[addresses.length - 1];
            // Search sequentially over the next 1000 ports
            for (int i = last.getPort() + 1; i < last.getPort() + 1001; i++) {
                int port = i;
                if (port > 65535) {
                    break;
                }
                try {
                    HostAndPort addr = HostAndPort.fromParts(hostname, port);
                    return TServerUtils.startTServer(serverType, timedProcessor, serverName, threadName, minThreads, simpleTimerThreadpoolSize, timeBetweenThreadChecks, maxMessageSize, service.getServerSslParams(), service.getSaslParams(), service.getClientTimeoutInMillis(), addr);
                } catch (TTransportException tte) {
                    log.info("Unable to use port {}, retrying. (Thread Name = {})", port, threadName);
                }
            }
            log.error("Unable to start TServer", e);
            throw new UnknownHostException("Unable to find a listen port");
        } else {
            log.error("Unable to start TServer", e);
            throw new UnknownHostException("Unable to find a listen port");
        }
    }
}

Example 19 with HostAndPort

use of org.apache.accumulo.core.util.HostAndPort in project accumulo by apache.

the class VerifyTabletAssignments method checkTable.

private static void checkTable(final ClientContext context, final Opts opts, String tableName, HashSet<KeyExtent> check) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, InterruptedException {
    if (check == null)
        System.out.println("Checking table " + tableName);
    else
        System.out.println("Checking table " + tableName + " again, failures " + check.size());
    TreeMap<KeyExtent, String> tabletLocations = new TreeMap<>();
    Table.ID tableId = Tables.getNameToIdMap(context.getInstance()).get(tableName);
    MetadataServicer.forTableId(context, tableId).getTabletLocations(tabletLocations);
    final HashSet<KeyExtent> failures = new HashSet<>();
    Map<HostAndPort, List<KeyExtent>> extentsPerServer = new TreeMap<>();
    for (Entry<KeyExtent, String> entry : tabletLocations.entrySet()) {
        KeyExtent keyExtent = entry.getKey();
        String loc = entry.getValue();
        if (loc == null)
            System.out.println(" Tablet " + keyExtent + " has no location");
        else if (opts.verbose)
            System.out.println(" Tablet " + keyExtent + " is located at " + loc);
        if (loc != null) {
            final HostAndPort parsedLoc = HostAndPort.fromString(loc);
            List<KeyExtent> extentList = extentsPerServer.get(parsedLoc);
            if (extentList == null) {
                extentList = new ArrayList<>();
                extentsPerServer.put(parsedLoc, extentList);
            }
            if (check == null || check.contains(keyExtent))
                extentList.add(keyExtent);
        }
    }
    ExecutorService tp = Executors.newFixedThreadPool(20);
    for (final Entry<HostAndPort, List<KeyExtent>> entry : extentsPerServer.entrySet()) {
        Runnable r = new Runnable() {

            @Override
            public void run() {
                try {
                    checkTabletServer(context, entry, failures);
                } catch (Exception e) {
                    log.error("Failure on tablet server '" + entry.getKey() + ".", e);
                    failures.addAll(entry.getValue());
                }
            }
        };
        tp.execute(r);
    }
    tp.shutdown();
    while (!tp.awaitTermination(1, TimeUnit.HOURS)) {
    }
    if (failures.size() > 0)
        checkTable(context, opts, tableName, failures);
}

Example 20 with HostAndPort

use of org.apache.accumulo.core.util.HostAndPort in project accumulo by apache.

the class LiveTServerSet method find.

TServerInstance find(Map<String, TServerInfo> servers, String tabletServer) {
    HostAndPort addr;
    String sessionId = null;
    if (']' == tabletServer.charAt(tabletServer.length() - 1)) {
        int index = tabletServer.indexOf('[');
        if (-1 == index) {
            throw new IllegalArgumentException("Could not parse tabletserver '" + tabletServer + "'");
        }
        addr = AddressUtil.parseAddress(tabletServer.substring(0, index), false);
        // Strip off the last bracket
        sessionId = tabletServer.substring(index + 1, tabletServer.length() - 1);
    } else {
        addr = AddressUtil.parseAddress(tabletServer, false);
    }
    for (Entry<String, TServerInfo> entry : servers.entrySet()) {
        if (entry.getValue().instance.getLocation().equals(addr)) {
            // Return the instance if we have no desired session ID, or we match the desired session ID
            if (null == sessionId || sessionId.equals(entry.getValue().instance.getSession()))
                return entry.getValue().instance;
        }
    }
    return null;
}