Examples with MasterMonitorInfo org.apache.accumulo.core.master.thrift.MasterMonitorInfo used on opensource projects

Example 16 with MasterMonitorInfo

use of org.apache.accumulo.core.master.thrift.MasterMonitorInfo in project accumulo by apache.

the class ListBulkCommand method execute.

@Override
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception {
    List<String> tservers;
    MasterMonitorInfo stats;
    MasterClientService.Iface client = null;
    Instance instance = shellState.getInstance();
    AccumuloServerContext context = new AccumuloServerContext(instance, new ServerConfigurationFactory(instance));
    while (true) {
        try {
            client = MasterClient.getConnectionWithRetry(context);
            stats = client.getMasterStats(Tracer.traceInfo(), context.rpcCreds());
            break;
        } catch (ThriftNotActiveServiceException e) {
            // Let it loop, fetching a new location
            sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        } finally {
            if (client != null)
                MasterClient.close(client);
        }
    }
    final boolean paginate = !cl.hasOption(disablePaginationOpt.getOpt());
    if (cl.hasOption(tserverOption.getOpt())) {
        tservers = new ArrayList<>();
        tservers.add(cl.getOptionValue(tserverOption.getOpt()));
    } else {
        tservers = Collections.emptyList();
    }
    shellState.printLines(new BulkImportListIterator(tservers, stats), paginate);
    return 0;
}

Example 17 with MasterMonitorInfo

use of org.apache.accumulo.core.master.thrift.MasterMonitorInfo in project accumulo by apache.

the class GetFileInfoBulkIT method test.

@Test
public void test() throws Exception {
    final Connector c = getConnector();
    getCluster().getClusterControl().kill(ServerType.GARBAGE_COLLECTOR, "localhost");
    final String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    // turn off compactions
    c.tableOperations().setProperty(tableName, Property.TABLE_MAJC_RATIO.getKey(), "2000");
    c.tableOperations().setProperty(tableName, Property.TABLE_FILE_MAX.getKey(), "2000");
    // splits to slow down bulk import
    SortedSet<Text> splits = new TreeSet<>();
    for (int i = 1; i < 0xf; i++) {
        splits.add(new Text(Integer.toHexString(i)));
    }
    c.tableOperations().addSplits(tableName, splits);
    MasterMonitorInfo stats = getCluster().getMasterMonitorInfo();
    assertEquals(1, stats.tServerInfo.size());
    log.info("Creating lots of bulk import files");
    final FileSystem fs = getCluster().getFileSystem();
    final Path basePath = getCluster().getTemporaryPath();
    CachedConfiguration.setInstance(fs.getConf());
    final Path base = new Path(basePath, "testBulkLoad" + tableName);
    fs.delete(base, true);
    fs.mkdirs(base);
    ExecutorService es = Executors.newFixedThreadPool(5);
    List<Future<Pair<String, String>>> futures = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        final int which = i;
        futures.add(es.submit(new Callable<Pair<String, String>>() {

            @Override
            public Pair<String, String> call() throws Exception {
                Path bulkFailures = new Path(base, "failures" + which);
                Path files = new Path(base, "files" + which);
                fs.mkdirs(bulkFailures);
                fs.mkdirs(files);
                for (int i = 0; i < 100; i++) {
                    FileSKVWriter writer = FileOperations.getInstance().newWriterBuilder().forFile(files.toString() + "/bulk_" + i + "." + RFile.EXTENSION, fs, fs.getConf()).withTableConfiguration(DefaultConfiguration.getInstance()).build();
                    writer.startDefaultLocalityGroup();
                    for (int j = 0x100; j < 0xfff; j += 3) {
                        writer.append(new Key(Integer.toHexString(j)), new Value(new byte[0]));
                    }
                    writer.close();
                }
                return new Pair<>(files.toString(), bulkFailures.toString());
            }
        }));
    }
    List<Pair<String, String>> dirs = new ArrayList<>();
    for (Future<Pair<String, String>> f : futures) {
        dirs.add(f.get());
    }
    log.info("Importing");
    long startOps = getOpts();
    long now = System.currentTimeMillis();
    List<Future<Object>> errs = new ArrayList<>();
    for (Pair<String, String> entry : dirs) {
        final String dir = entry.getFirst();
        final String err = entry.getSecond();
        errs.add(es.submit(new Callable<Object>() {

            @Override
            public Object call() throws Exception {
                c.tableOperations().importDirectory(tableName, dir, err, false);
                return null;
            }
        }));
    }
    for (Future<Object> err : errs) {
        err.get();
    }
    es.shutdown();
    es.awaitTermination(2, TimeUnit.MINUTES);
    log.info(String.format("Completed in %.2f seconds", (System.currentTimeMillis() - now) / 1000.));
    sleepUninterruptibly(30, TimeUnit.SECONDS);
    long getFileInfoOpts = getOpts() - startOps;
    log.info("# opts: {}", getFileInfoOpts);
    assertTrue("unexpected number of getFileOps", getFileInfoOpts < 2100 && getFileInfoOpts > 1000);
}

Example 18 with MasterMonitorInfo

use of org.apache.accumulo.core.master.thrift.MasterMonitorInfo in project accumulo by apache.

the class MetadataMaxFilesIT method test.

@Test
public void test() throws Exception {
    Connector c = getConnector();
    SortedSet<Text> splits = new TreeSet<>();
    for (int i = 0; i < 1000; i++) {
        splits.add(new Text(String.format("%03d", i)));
    }
    c.tableOperations().setProperty(MetadataTable.NAME, Property.TABLE_SPLIT_THRESHOLD.getKey(), "10000");
    // propagation time
    sleepUninterruptibly(5, TimeUnit.SECONDS);
    for (int i = 0; i < 5; i++) {
        String tableName = "table" + i;
        log.info("Creating {}", tableName);
        c.tableOperations().create(tableName);
        log.info("adding splits");
        c.tableOperations().addSplits(tableName, splits);
        log.info("flushing");
        c.tableOperations().flush(MetadataTable.NAME, null, null, true);
        c.tableOperations().flush(RootTable.NAME, null, null, true);
    }
    log.info("shutting down");
    assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
    cluster.stop();
    log.info("starting up");
    cluster.start();
    Credentials creds = new Credentials("root", new PasswordToken(ROOT_PASSWORD));
    while (true) {
        MasterMonitorInfo stats = null;
        Client client = null;
        try {
            ClientContext context = new ClientContext(c.getInstance(), creds, getClientConfig());
            client = MasterClient.getConnectionWithRetry(context);
            log.info("Fetching stats");
            stats = client.getMasterStats(Tracer.traceInfo(), context.rpcCreds());
        } catch (ThriftNotActiveServiceException e) {
            // Let it loop, fetching a new location
            sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
            continue;
        } finally {
            if (client != null)
                MasterClient.close(client);
        }
        int tablets = 0;
        for (TabletServerStatus tserver : stats.tServerInfo) {
            for (Entry<String, TableInfo> entry : tserver.tableMap.entrySet()) {
                if (entry.getKey().startsWith("!") || entry.getKey().startsWith("+"))
                    continue;
                tablets += entry.getValue().onlineTablets;
            }
        }
        log.info("Online tablets " + tablets);
        if (tablets == 5005)
            break;
        sleepUninterruptibly(1, TimeUnit.SECONDS);
    }
}

Example 19 with MasterMonitorInfo

use of org.apache.accumulo.core.master.thrift.MasterMonitorInfo in project accumulo by apache.

the class SimpleBalancerFairnessIT method simpleBalancerFairness.

@Test
public void simpleBalancerFairness() throws Exception {
    Connector c = getConnector();
    c.tableOperations().create("test_ingest");
    c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
    c.tableOperations().create("unused");
    TreeSet<Text> splits = TestIngest.getSplitPoints(0, 10000000, 500);
    log.info("Creating {} splits", splits.size());
    c.tableOperations().addSplits("unused", splits);
    List<String> tservers = c.instanceOperations().getTabletServers();
    TestIngest.Opts opts = new TestIngest.Opts();
    opts.rows = 50000;
    opts.setPrincipal("root");
    TestIngest.ingest(c, opts, new BatchWriterOpts());
    c.tableOperations().flush("test_ingest", null, null, false);
    sleepUninterruptibly(45, TimeUnit.SECONDS);
    Credentials creds = new Credentials("root", new PasswordToken(ROOT_PASSWORD));
    ClientContext context = new ClientContext(c.getInstance(), creds, getClientConfig());
    MasterMonitorInfo stats = null;
    int unassignedTablets = 1;
    for (int i = 0; unassignedTablets > 0 && i < 10; i++) {
        MasterClientService.Iface client = null;
        while (true) {
            try {
                client = MasterClient.getConnectionWithRetry(context);
                stats = client.getMasterStats(Tracer.traceInfo(), creds.toThrift(c.getInstance()));
                break;
            } catch (ThriftNotActiveServiceException e) {
                // Let it loop, fetching a new location
                sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
            } finally {
                if (client != null)
                    MasterClient.close(client);
            }
        }
        unassignedTablets = stats.getUnassignedTablets();
        if (unassignedTablets > 0) {
            log.info("Found {} unassigned tablets, sleeping 3 seconds for tablet assignment", unassignedTablets);
            Thread.sleep(3000);
        }
    }
    assertEquals("Unassigned tablets were not assigned within 30 seconds", 0, unassignedTablets);
    // Compute online tablets per tserver
    List<Integer> counts = new ArrayList<>();
    for (TabletServerStatus server : stats.tServerInfo) {
        int count = 0;
        for (TableInfo table : server.tableMap.values()) {
            count += table.onlineTablets;
        }
        counts.add(count);
    }
    assertTrue("Expected to have at least two TabletServers", counts.size() > 1);
    for (int i = 1; i < counts.size(); i++) {
        int diff = Math.abs(counts.get(0) - counts.get(i));
        assertTrue("Expected difference in tablets to be less than or equal to " + counts.size() + " but was " + diff + ". Counts " + counts, diff <= tservers.size());
    }
}