Examples with HostAndPort org.apache.accumulo.core.util.HostAndPort used on opensource projects

Example 31 with HostAndPort

use of org.apache.accumulo.core.util.HostAndPort in project accumulo by apache.

the class SuspendedTabletsIT method suspensionTestBody.

/**
 * Main test body for suspension tests.
 *
 * @param serverStopper
 *          callback which shuts down some tablet servers.
 */
private void suspensionTestBody(TServerKiller serverStopper) throws Exception {
    Credentials creds = new Credentials("root", new PasswordToken(ROOT_PASSWORD));
    Instance instance = new ZooKeeperInstance(getCluster().getClientConfig());
    ClientContext ctx = new ClientContext(instance, creds, getCluster().getClientConfig());
    String tableName = getUniqueNames(1)[0];
    Connector conn = ctx.getConnector();
    // Create a table with a bunch of splits
    log.info("Creating table " + tableName);
    conn.tableOperations().create(tableName);
    SortedSet<Text> splitPoints = new TreeSet<>();
    for (int i = 1; i < TABLETS; ++i) {
        splitPoints.add(new Text("" + i));
    }
    conn.tableOperations().addSplits(tableName, splitPoints);
    // Wait for all of the tablets to hosted ...
    log.info("Waiting on hosting and balance");
    TabletLocations ds;
    for (ds = TabletLocations.retrieve(ctx, tableName); ds.hostedCount != TABLETS; ds = TabletLocations.retrieve(ctx, tableName)) {
        Thread.sleep(1000);
    }
    // ... and balanced.
    conn.instanceOperations().waitForBalance();
    do {
        // Give at least another 5 seconds for migrations to finish up
        Thread.sleep(5000);
        ds = TabletLocations.retrieve(ctx, tableName);
    } while (ds.hostedCount != TABLETS);
    // Pray all of our tservers have at least 1 tablet.
    Assert.assertEquals(TSERVERS, ds.hosted.keySet().size());
    // Kill two tablet servers hosting our tablets. This should put tablets into suspended state, and thus halt balancing.
    TabletLocations beforeDeathState = ds;
    log.info("Eliminating tablet servers");
    serverStopper.eliminateTabletServers(ctx, beforeDeathState, 2);
    // Eventually some tablets will be suspended.
    log.info("Waiting on suspended tablets");
    ds = TabletLocations.retrieve(ctx, tableName);
    // Until we can scan the metadata table, the master probably can't either, so won't have been able to suspend the tablets.
    // So we note the time that we were first able to successfully scan the metadata table.
    long killTime = System.nanoTime();
    while (ds.suspended.keySet().size() != 2) {
        Thread.sleep(1000);
        ds = TabletLocations.retrieve(ctx, tableName);
    }
    SetMultimap<HostAndPort, KeyExtent> deadTabletsByServer = ds.suspended;
    // "belong" to the dead tablet servers, and should be in exactly the same place as before any tserver death.
    for (HostAndPort server : deadTabletsByServer.keySet()) {
        Assert.assertEquals(deadTabletsByServer.get(server), beforeDeathState.hosted.get(server));
    }
    Assert.assertEquals(TABLETS, ds.hostedCount + ds.suspendedCount);
    // Restart the first tablet server, making sure it ends up on the same port
    HostAndPort restartedServer = deadTabletsByServer.keySet().iterator().next();
    log.info("Restarting " + restartedServer);
    getCluster().getClusterControl().start(ServerType.TABLET_SERVER, null, ImmutableMap.of(Property.TSERV_CLIENTPORT.getKey(), "" + restartedServer.getPort(), Property.TSERV_PORTSEARCH.getKey(), "false"), 1);
    // Eventually, the suspended tablets should be reassigned to the newly alive tserver.
    log.info("Awaiting tablet unsuspension for tablets belonging to " + restartedServer);
    for (ds = TabletLocations.retrieve(ctx, tableName); ds.suspended.containsKey(restartedServer) || ds.assignedCount != 0; ds = TabletLocations.retrieve(ctx, tableName)) {
        Thread.sleep(1000);
    }
    Assert.assertEquals(deadTabletsByServer.get(restartedServer), ds.hosted.get(restartedServer));
    // Finally, after much longer, remaining suspended tablets should be reassigned.
    log.info("Awaiting tablet reassignment for remaining tablets");
    for (ds = TabletLocations.retrieve(ctx, tableName); ds.hostedCount != TABLETS; ds = TabletLocations.retrieve(ctx, tableName)) {
        Thread.sleep(1000);
    }
    long recoverTime = System.nanoTime();
    Assert.assertTrue(recoverTime - killTime >= NANOSECONDS.convert(SUSPEND_DURATION, MILLISECONDS));
}

Example 32 with HostAndPort

use of org.apache.accumulo.core.util.HostAndPort in project accumulo by apache.

the class NullTserver method main.

public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    opts.parseArgs(NullTserver.class.getName(), args);
    // modify metadata
    ZooKeeperInstance zki = new ZooKeeperInstance(ClientConfiguration.create().withInstance(opts.iname).withZkHosts(opts.keepers));
    Instance inst = HdfsZooInstance.getInstance();
    AccumuloServerContext context = new AccumuloServerContext(inst, new ServerConfigurationFactory(zki));
    TransactionWatcher watcher = new TransactionWatcher();
    ThriftClientHandler tch = new ThriftClientHandler(new AccumuloServerContext(inst, new ServerConfigurationFactory(inst)), watcher);
    Processor<Iface> processor = new Processor<>(tch);
    TServerUtils.startTServer(context.getConfiguration(), ThriftServerType.CUSTOM_HS_HA, processor, "NullTServer", "null tserver", 2, 1, 1000, 10 * 1024 * 1024, null, null, -1, HostAndPort.fromParts("0.0.0.0", opts.port));
    HostAndPort addr = HostAndPort.fromParts(InetAddress.getLocalHost().getHostName(), opts.port);
    Table.ID tableId = Tables.getTableId(zki, opts.tableName);
    // read the locations for the table
    Range tableRange = new KeyExtent(tableId, null, null).toMetadataRange();
    List<Assignment> assignments = new ArrayList<>();
    try (MetaDataTableScanner s = new MetaDataTableScanner(context, tableRange)) {
        long randomSessionID = opts.port;
        TServerInstance instance = new TServerInstance(addr, randomSessionID);
        while (s.hasNext()) {
            TabletLocationState next = s.next();
            assignments.add(new Assignment(next.extent, instance));
        }
    }
    // point them to this server
    MetaDataStateStore store = new MetaDataStateStore(context);
    store.setLocations(assignments);
    while (true) {
        sleepUninterruptibly(10, TimeUnit.SECONDS);
    }
}

Example 33 with HostAndPort

use of org.apache.accumulo.core.util.HostAndPort in project accumulo by apache.

the class AddressUtil method parseAddress.

public static HostAndPort parseAddress(String address, boolean ignoreMissingPort) throws NumberFormatException {
    address = address.replace('+', ':');
    HostAndPort hap = HostAndPort.fromString(address);
    if (!ignoreMissingPort && !hap.hasPort())
        throw new IllegalArgumentException("Address was expected to contain port. address=" + address);
    return hap;
}

Example 34 with HostAndPort

use of org.apache.accumulo.core.util.HostAndPort in project accumulo by apache.

the class ThriftScanner method getBatchFromServer.

public static boolean getBatchFromServer(ClientContext context, Range range, KeyExtent extent, String server, SortedMap<Key, Value> results, SortedSet<Column> fetchedColumns, List<IterInfo> serverSideIteratorList, Map<String, Map<String, String>> serverSideIteratorOptions, int size, Authorizations authorizations, boolean retry, long batchTimeOut, String classLoaderContext) throws AccumuloException, AccumuloSecurityException, NotServingTabletException {
    if (server == null)
        throw new AccumuloException(new IOException());
    final HostAndPort parsedServer = HostAndPort.fromString(server);
    try {
        TInfo tinfo = Tracer.traceInfo();
        TabletClientService.Client client = ThriftUtil.getTServerClient(parsedServer, context);
        try {
            // not reading whole rows (or stopping on row boundries) so there is no need to enable isolation below
            ScanState scanState = new ScanState(context, extent.getTableId(), authorizations, range, fetchedColumns, size, serverSideIteratorList, serverSideIteratorOptions, false, Constants.SCANNER_DEFAULT_READAHEAD_THRESHOLD, null, batchTimeOut, classLoaderContext);
            TabletType ttype = TabletType.type(extent);
            boolean waitForWrites = !serversWaitedForWrites.get(ttype).contains(server);
            InitialScan isr = client.startScan(tinfo, scanState.context.rpcCreds(), extent.toThrift(), scanState.range.toThrift(), Translator.translate(scanState.columns, Translators.CT), scanState.size, scanState.serverSideIteratorList, scanState.serverSideIteratorOptions, scanState.authorizations.getAuthorizationsBB(), waitForWrites, scanState.isolated, scanState.readaheadThreshold, null, scanState.batchTimeOut, classLoaderContext);
            if (waitForWrites)
                serversWaitedForWrites.get(ttype).add(server);
            Key.decompress(isr.result.results);
            for (TKeyValue kv : isr.result.results) results.put(new Key(kv.key), new Value(kv.value));
            client.closeScan(tinfo, isr.scanID);
            return isr.result.more;
        } finally {
            ThriftUtil.returnClient(client);
        }
    } catch (TApplicationException tae) {
        throw new AccumuloServerException(server, tae);
    } catch (TooManyFilesException e) {
        log.debug("Tablet ({}) has too many files {} : {}", extent, server, e.getMessage());
    } catch (ThriftSecurityException e) {
        log.warn("Security Violation in scan request to {}: {}", server, e.getMessage());
        throw new AccumuloSecurityException(e.user, e.code, e);
    } catch (TException e) {
        log.debug("Error getting transport to {}: {}", server, e.getMessage());
    }
    throw new AccumuloException("getBatchFromServer: failed");
}

Example 35 with HostAndPort

use of org.apache.accumulo.core.util.HostAndPort in project accumulo by apache.

the class ThriftScanner method scan.

private static List<KeyValue> scan(TabletLocation loc, ScanState scanState, ClientContext context) throws AccumuloSecurityException, NotServingTabletException, TException, NoSuchScanIDException, TooManyFilesException, TSampleNotPresentException {
    if (scanState.finished)
        return null;
    OpTimer timer = null;
    final TInfo tinfo = Tracer.traceInfo();
    final HostAndPort parsedLocation = HostAndPort.fromString(loc.tablet_location);
    TabletClientService.Client client = ThriftUtil.getTServerClient(parsedLocation, context);
    String old = Thread.currentThread().getName();
    try {
        ScanResult sr;
        if (scanState.prevLoc != null && !scanState.prevLoc.equals(loc))
            scanState.scanID = null;
        scanState.prevLoc = loc;
        if (scanState.scanID == null) {
            Thread.currentThread().setName("Starting scan tserver=" + loc.tablet_location + " tableId=" + loc.tablet_extent.getTableId());
            if (log.isTraceEnabled()) {
                String msg = "Starting scan tserver=" + loc.tablet_location + " tablet=" + loc.tablet_extent + " range=" + scanState.range + " ssil=" + scanState.serverSideIteratorList + " ssio=" + scanState.serverSideIteratorOptions + " context=" + scanState.classLoaderContext;
                log.trace("tid={} {}", Thread.currentThread().getId(), msg);
                timer = new OpTimer().start();
            }
            TabletType ttype = TabletType.type(loc.tablet_extent);
            boolean waitForWrites = !serversWaitedForWrites.get(ttype).contains(loc.tablet_location);
            InitialScan is = client.startScan(tinfo, scanState.context.rpcCreds(), loc.tablet_extent.toThrift(), scanState.range.toThrift(), Translator.translate(scanState.columns, Translators.CT), scanState.size, scanState.serverSideIteratorList, scanState.serverSideIteratorOptions, scanState.authorizations.getAuthorizationsBB(), waitForWrites, scanState.isolated, scanState.readaheadThreshold, SamplerConfigurationImpl.toThrift(scanState.samplerConfig), scanState.batchTimeOut, scanState.classLoaderContext);
            if (waitForWrites)
                serversWaitedForWrites.get(ttype).add(loc.tablet_location);
            sr = is.result;
            if (sr.more)
                scanState.scanID = is.scanID;
            else
                client.closeScan(tinfo, is.scanID);
        } else {
            // log.debug("Calling continue scan : "+scanState.range+" loc = "+loc);
            String msg = "Continuing scan tserver=" + loc.tablet_location + " scanid=" + scanState.scanID;
            Thread.currentThread().setName(msg);
            if (log.isTraceEnabled()) {
                log.trace("tid={} {}", Thread.currentThread().getId(), msg);
                timer = new OpTimer().start();
            }
            sr = client.continueScan(tinfo, scanState.scanID);
            if (!sr.more) {
                client.closeScan(tinfo, scanState.scanID);
                scanState.scanID = null;
            }
        }
        if (!sr.more) {
            // log.debug("No more : tab end row = "+loc.tablet_extent.getEndRow()+" range = "+scanState.range);
            if (loc.tablet_extent.getEndRow() == null) {
                scanState.finished = true;
                if (timer != null) {
                    timer.stop();
                    log.trace("tid={} Completely finished scan in {} #results={}", Thread.currentThread().getId(), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)), sr.results.size());
                }
            } else if (scanState.range.getEndKey() == null || !scanState.range.afterEndKey(new Key(loc.tablet_extent.getEndRow()).followingKey(PartialKey.ROW))) {
                scanState.startRow = loc.tablet_extent.getEndRow();
                scanState.skipStartRow = true;
                if (timer != null) {
                    timer.stop();
                    log.trace("tid={} Finished scanning tablet in {} #results={}", Thread.currentThread().getId(), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)), sr.results.size());
                }
            } else {
                scanState.finished = true;
                if (timer != null) {
                    timer.stop();
                    log.trace("tid={} Completely finished in {} #results={}", Thread.currentThread().getId(), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)), sr.results.size());
                }
            }
        } else {
            if (timer != null) {
                timer.stop();
                log.trace("tid={} Finished scan in {} #results={} scanid={}", Thread.currentThread().getId(), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)), sr.results.size(), scanState.scanID);
            }
        }
        Key.decompress(sr.results);
        if (sr.results.size() > 0 && !scanState.finished)
            scanState.range = new Range(new Key(sr.results.get(sr.results.size() - 1).key), false, scanState.range.getEndKey(), scanState.range.isEndKeyInclusive());
        List<KeyValue> results = new ArrayList<>(sr.results.size());
        for (TKeyValue tkv : sr.results) results.add(new KeyValue(new Key(tkv.key), tkv.value));
        return results;
    } catch (ThriftSecurityException e) {
        throw new AccumuloSecurityException(e.user, e.code, e);
    } finally {
        ThriftUtil.returnClient(client);
        Thread.currentThread().setName(old);
    }
}