Examples with ManagerMonitorInfo org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo used on opensource projects

Example 6 with ManagerMonitorInfo

use of org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo in project accumulo by apache.

the class TabletServerResource method getTserverRecovery.

/**
 * Generates a recovery tserver list
 *
 * @return Recovery tserver list
 */
@Path("recovery")
@GET
public TabletServersRecovery getTserverRecovery() {
    TabletServersRecovery recoveryList = new TabletServersRecovery();
    ManagerMonitorInfo mmi = monitor.getMmi();
    if (mmi == null) {
        return new TabletServersRecovery();
    }
    for (TabletServerStatus server : mmi.tServerInfo) {
        if (server.logSorts != null) {
            for (RecoveryStatus recovery : server.logSorts) {
                String serv = AddressUtil.parseAddress(server.name, false).getHost();
                String log = recovery.name;
                int time = recovery.runtime;
                double progress = recovery.progress;
                recoveryList.addRecovery(new TabletServerRecoveryInformation(serv, log, time, progress));
            }
        }
    }
    return recoveryList;
}

Example 7 with ManagerMonitorInfo

use of org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo in project accumulo by apache.

the class ManagerResource method getNumBadTservers.

/**
 * Generates bad tserver lists as a JSON object
 *
 * @return bad tserver list
 */
public static BadTabletServers getNumBadTservers(Monitor monitor) {
    ManagerMonitorInfo mmi = monitor.getMmi();
    if (mmi == null) {
        return new BadTabletServers();
    }
    Map<String, Byte> badServers = mmi.getBadTServers();
    if (badServers == null || badServers.isEmpty()) {
        return new BadTabletServers();
    }
    BadTabletServers readableBadServers = new BadTabletServers();
    // Add new bad tservers to the list
    for (Entry<String, Byte> badServer : badServers.entrySet()) {
        try {
            TabletServerState state = TabletServerState.getStateById(badServer.getValue());
            readableBadServers.addBadServer(new BadTabletServerInformation(badServer.getKey(), state.name()));
        } catch (IndexOutOfBoundsException e) {
            readableBadServers.addBadServer(new BadTabletServerInformation(badServer.getKey(), "Unknown state"));
        }
    }
    return readableBadServers;
}

Example 8 with ManagerMonitorInfo

use of org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo in project accumulo by apache.

the class StatusResource method getTables.

/**
 * Generates the JSON object with the status
 *
 * @return Status report
 */
@GET
public StatusInformation getTables() {
    Status managerStatus;
    Status gcStatus;
    Status tServerStatus = Status.ERROR;
    ManagerMonitorInfo mmi = monitor.getMmi();
    if (mmi != null) {
        if (monitor.getGcStatus() != null) {
            gcStatus = Status.OK;
        } else {
            gcStatus = Status.ERROR;
        }
        List<String> managers = monitor.getContext().getManagerLocations();
        managerStatus = managers.isEmpty() ? Status.ERROR : Status.OK;
        int tServerUp = mmi.getTServerInfoSize();
        int tServerDown = mmi.getDeadTabletServersSize();
        int tServerBad = mmi.getBadTServersSize();
        /*
       * If there are no dead or bad servers and there are tservers up, status is OK, if there are
       * dead or bad servers and there is at least a tserver up, status is WARN, otherwise, the
       * status is an error.
       */
        if ((tServerDown > 0 || tServerBad > 0) && tServerUp > 0) {
            tServerStatus = Status.WARN;
        } else if ((tServerDown == 0 || tServerBad == 0) && tServerUp > 0) {
            tServerStatus = Status.OK;
        } else if (tServerUp == 0) {
            tServerStatus = Status.ERROR;
        }
    } else {
        managerStatus = Status.ERROR;
        if (monitor.getGcStatus() == null) {
            gcStatus = Status.ERROR;
        } else {
            gcStatus = Status.OK;
        }
        tServerStatus = Status.ERROR;
    }
    return new StatusInformation(managerStatus.toString(), gcStatus.toString(), tServerStatus.toString(), monitor.recentLogs().numEvents(), monitor.recentLogs().eventsIncludeErrors(), monitor.getProblemSummary().entrySet().size());
}

Example 9 with ManagerMonitorInfo

use of org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo in project accumulo by apache.

the class CountNameNodeOpsBulkIT method compareOldNewBulkImportTest.

@Test
public void compareOldNewBulkImportTest() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProperties()).build()) {
        getCluster().getClusterControl().kill(ServerType.GARBAGE_COLLECTOR, "localhost");
        final String tableName = getUniqueNames(1)[0];
        // disable compactions
        Map<String, String> props = new HashMap<>();
        props.put(Property.TABLE_MAJC_RATIO.getKey(), "2000");
        props.put(Property.TABLE_FILE_MAX.getKey(), "2000");
        // splits to slow down bulk import
        SortedSet<Text> splits = new TreeSet<>();
        for (int i = 1; i < 0xf; i++) {
            splits.add(new Text(Integer.toHexString(i)));
        }
        var ntc = new NewTableConfiguration().setProperties(props).withSplits(splits);
        c.tableOperations().create(tableName, ntc);
        ManagerMonitorInfo stats = getCluster().getManagerMonitorInfo();
        assertEquals(1, stats.tServerInfo.size());
        log.info("Creating lots of bulk import files");
        final FileSystem fs = getCluster().getFileSystem();
        final Path basePath = getCluster().getTemporaryPath();
        final Path base = new Path(basePath, "testBulkLoad" + tableName);
        fs.delete(base, true);
        fs.mkdirs(base);
        ExecutorService es = Executors.newFixedThreadPool(5);
        List<Future<String>> futures = new ArrayList<>();
        for (int i = 0; i < 10; i++) {
            final int which = i;
            futures.add(es.submit(() -> {
                Path files = new Path(base, "files" + which);
                fs.mkdirs(files);
                for (int i1 = 0; i1 < 100; i1++) {
                    FileSKVWriter writer = FileOperations.getInstance().newWriterBuilder().forFile(files + "/bulk_" + i1 + "." + RFile.EXTENSION, fs, fs.getConf(), CryptoServiceFactory.newDefaultInstance()).withTableConfiguration(DefaultConfiguration.getInstance()).build();
                    writer.startDefaultLocalityGroup();
                    for (int j = 0x100; j < 0xfff; j += 3) {
                        writer.append(new Key(Integer.toHexString(j)), new Value());
                    }
                    writer.close();
                }
                return files.toString();
            }));
        }
        List<String> dirs = new ArrayList<>();
        for (Future<String> f : futures) {
            dirs.add(f.get());
        }
        log.info("Importing");
        long startOps = getStat(getStats(), "FileInfoOps");
        long now = System.currentTimeMillis();
        List<Future<Object>> errs = new ArrayList<>();
        for (String dir : dirs) {
            errs.add(es.submit(() -> {
                c.tableOperations().importDirectory(dir).to(tableName).load();
                return null;
            }));
        }
        for (Future<Object> err : errs) {
            err.get();
        }
        es.shutdown();
        es.awaitTermination(2, TimeUnit.MINUTES);
        log.info(String.format("Completed in %.2f seconds", (System.currentTimeMillis() - now) / 1000.));
        sleepUninterruptibly(30, TimeUnit.SECONDS);
        Map<?, ?> map = getStats();
        map.forEach((k, v) -> {
            try {
                if (v != null && Double.parseDouble(v.toString()) > 0.0)
                    log.debug("{}:{}", k, v);
            } catch (NumberFormatException e) {
            // only looking for numbers
            }
        });
        long getFileInfoOpts = getStat(map, "FileInfoOps") - startOps;
        log.info("New bulk import used {} opts, vs old using 2060", getFileInfoOpts);
        // counts for old bulk import:
        // Expected number of FileInfoOps was between 1000 and 2100
        // new bulk import is way better :)
        assertEquals("unexpected number of FileInfoOps", 20, getFileInfoOpts);
    }
}

Example 10 with ManagerMonitorInfo

use of org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo in project accumulo by apache.

the class DetectDeadTabletServersIT method test.

@Test
public void test() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProperties()).build()) {
        log.info("verifying that everything is up");
        Iterators.size(c.createScanner(MetadataTable.NAME, Authorizations.EMPTY).iterator());
        ManagerMonitorInfo stats = getStats(c);
        assertEquals(2, stats.tServerInfo.size());
        assertEquals(0, stats.badTServers.size());
        assertEquals(0, stats.deadTabletServers.size());
        log.info("Killing a tablet server");
        getCluster().killProcess(TABLET_SERVER, getCluster().getProcesses().get(TABLET_SERVER).iterator().next());
        while (true) {
            stats = getStats(c);
            if (stats.tServerInfo.size() != 2) {
                break;
            }
            UtilWaitThread.sleep(500);
        }
        assertEquals(1, stats.tServerInfo.size());
        assertEquals(1, stats.badTServers.size() + stats.deadTabletServers.size());
        while (true) {
            stats = getStats(c);
            if (!stats.deadTabletServers.isEmpty()) {
                break;
            }
            UtilWaitThread.sleep(500);
        }
        assertEquals(1, stats.tServerInfo.size());
        assertEquals(0, stats.badTServers.size());
        assertEquals(1, stats.deadTabletServers.size());
    }
}