Examples with TableInfo org.apache.accumulo.core.master.thrift.TableInfo used on opensource projects

Example 6 with TableInfo

use of org.apache.accumulo.core.master.thrift.TableInfo in project accumulo by apache.

the class ChaoticLoadBalancer method getAssignments.

@Override
public void getAssignments(SortedMap<TServerInstance, TabletServerStatus> current, Map<KeyExtent, TServerInstance> unassigned, Map<KeyExtent, TServerInstance> assignments) {
    long total = assignments.size() + unassigned.size();
    long avg = (long) Math.ceil(((double) total) / current.size());
    Map<TServerInstance, Long> toAssign = new HashMap<>();
    List<TServerInstance> tServerArray = new ArrayList<>();
    for (Entry<TServerInstance, TabletServerStatus> e : current.entrySet()) {
        long numTablets = 0;
        for (TableInfo ti : e.getValue().getTableMap().values()) {
            numTablets += ti.tablets;
        }
        if (numTablets <= avg) {
            tServerArray.add(e.getKey());
            toAssign.put(e.getKey(), avg - numTablets);
        }
    }
    if (tServerArray.isEmpty()) {
        // No tservers to assign to
        return;
    }
    for (KeyExtent ke : unassigned.keySet()) {
        int index = r.nextInt(tServerArray.size());
        TServerInstance dest = tServerArray.get(index);
        assignments.put(ke, dest);
        long remaining = toAssign.get(dest) - 1;
        if (remaining == 0) {
            tServerArray.remove(index);
            toAssign.remove(dest);
        } else {
            toAssign.put(dest, remaining);
        }
    }
}

Example 7 with TableInfo

use of org.apache.accumulo.core.master.thrift.TableInfo in project accumulo by apache.

the class DefaultLoadBalancer method busiest.

// define what it means for a tablet to be busy
private static Table.ID busiest(Map<String, TableInfo> tables) {
    Table.ID result = null;
    double busiest = Double.NEGATIVE_INFINITY;
    for (Entry<String, TableInfo> entry : tables.entrySet()) {
        TableInfo info = entry.getValue();
        double busy = info.ingestRate + info.queryRate;
        if (busy > busiest) {
            busiest = busy;
            result = Table.ID.of(entry.getKey());
        }
    }
    return result;
}

Example 8 with TableInfo

use of org.apache.accumulo.core.master.thrift.TableInfo in project accumulo by apache.

the class Monitor method fetchData.

public static void fetchData() {
    double totalIngestRate = 0.;
    double totalIngestByteRate = 0.;
    double totalQueryRate = 0.;
    double totalQueryByteRate = 0.;
    double totalScanRate = 0.;
    long totalEntries = 0;
    int totalTabletCount = 0;
    long totalHoldTime = 0;
    long totalLookups = 0;
    boolean retry = true;
    // only recalc every so often
    long currentTime = System.currentTimeMillis();
    if (currentTime - lastRecalc.get() < REFRESH_TIME * 1000)
        return;
    synchronized (Monitor.class) {
        // Learn our instance name asynchronously so we don't hang up if zookeeper is down
        if (cachedInstanceName.get().equals(DEFAULT_INSTANCE_NAME)) {
            SimpleTimer.getInstance(config.getSystemConfiguration()).schedule(new TimerTask() {

                @Override
                public void run() {
                    synchronized (Monitor.class) {
                        if (cachedInstanceName.get().equals(DEFAULT_INSTANCE_NAME)) {
                            final String instanceName = HdfsZooInstance.getInstance().getInstanceName();
                            if (null != instanceName) {
                                cachedInstanceName.set(instanceName);
                            }
                        }
                    }
                }
            }, 0);
        }
    }
    synchronized (Monitor.class) {
        if (fetching)
            return;
        fetching = true;
    }
    try {
        while (retry) {
            MasterClientService.Iface client = null;
            try {
                client = MasterClient.getConnection(context);
                if (client != null) {
                    mmi = client.getMasterStats(Tracer.traceInfo(), context.rpcCreds());
                    retry = false;
                } else {
                    mmi = null;
                }
                Monitor.gcStatus = fetchGcStatus();
            } catch (Exception e) {
                mmi = null;
                log.info("Error fetching stats: ", e);
            } finally {
                if (client != null) {
                    MasterClient.close(client);
                }
            }
            if (mmi == null)
                sleepUninterruptibly(1, TimeUnit.SECONDS);
        }
        if (mmi != null) {
            int majorCompactions = 0;
            int minorCompactions = 0;
            lookupRateTracker.startingUpdates();
            indexCacheHitTracker.startingUpdates();
            indexCacheRequestTracker.startingUpdates();
            dataCacheHitTracker.startingUpdates();
            dataCacheRequestTracker.startingUpdates();
            for (TabletServerStatus server : mmi.tServerInfo) {
                TableInfo summary = TableInfoUtil.summarizeTableStats(server);
                totalIngestRate += summary.ingestRate;
                totalIngestByteRate += summary.ingestByteRate;
                totalQueryRate += summary.queryRate;
                totalScanRate += summary.scanRate;
                totalQueryByteRate += summary.queryByteRate;
                totalEntries += summary.recs;
                totalHoldTime += server.holdTime;
                totalLookups += server.lookups;
                majorCompactions += summary.majors.running;
                minorCompactions += summary.minors.running;
                lookupRateTracker.updateTabletServer(server.name, server.lastContact, server.lookups);
                indexCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheHits);
                indexCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheRequest);
                dataCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheHits);
                dataCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheRequest);
            }
            lookupRateTracker.finishedUpdating();
            indexCacheHitTracker.finishedUpdating();
            indexCacheRequestTracker.finishedUpdating();
            dataCacheHitTracker.finishedUpdating();
            dataCacheRequestTracker.finishedUpdating();
            int totalTables = 0;
            for (TableInfo tInfo : mmi.tableMap.values()) {
                totalTabletCount += tInfo.tablets;
                totalTables++;
            }
            Monitor.totalIngestRate = totalIngestRate;
            Monitor.totalTables = totalTables;
            totalIngestByteRate = totalIngestByteRate / 1000000.0;
            Monitor.totalQueryRate = totalQueryRate;
            Monitor.totalScanRate = totalScanRate;
            totalQueryByteRate = totalQueryByteRate / 1000000.0;
            Monitor.totalEntries = totalEntries;
            Monitor.totalTabletCount = totalTabletCount;
            Monitor.totalHoldTime = totalHoldTime;
            Monitor.totalLookups = totalLookups;
            ingestRateOverTime.add(new Pair<>(currentTime, totalIngestRate));
            ingestByteRateOverTime.add(new Pair<>(currentTime, totalIngestByteRate));
            double totalLoad = 0.;
            for (TabletServerStatus status : mmi.tServerInfo) {
                if (status != null)
                    totalLoad += status.osLoad;
            }
            loadOverTime.add(new Pair<>(currentTime, totalLoad));
            minorCompactionsOverTime.add(new Pair<>(currentTime, minorCompactions));
            majorCompactionsOverTime.add(new Pair<>(currentTime, majorCompactions));
            lookupsOverTime.add(new Pair<>(currentTime, lookupRateTracker.calculateRate()));
            queryRateOverTime.add(new Pair<>(currentTime, (int) totalQueryRate));
            queryByteRateOverTime.add(new Pair<>(currentTime, totalQueryByteRate));
            scanRateOverTime.add(new Pair<>(currentTime, (int) totalScanRate));
            calcCacheHitRate(indexCacheHitRateOverTime, currentTime, indexCacheHitTracker, indexCacheRequestTracker);
            calcCacheHitRate(dataCacheHitRateOverTime, currentTime, dataCacheHitTracker, dataCacheRequestTracker);
        }
        try {
            Monitor.problemSummary = ProblemReports.getInstance(getContext()).summarize();
            Monitor.problemException = null;
        } catch (Exception e) {
            log.info("Failed to obtain problem reports ", e);
            Monitor.problemSummary = Collections.emptyMap();
            Monitor.problemException = e;
        }
    } finally {
        synchronized (Monitor.class) {
            fetching = false;
            lastRecalc.set(currentTime);
        }
    }
}

Example 9 with TableInfo

use of org.apache.accumulo.core.master.thrift.TableInfo in project accumulo by apache.

the class TablesResource method getTables.

/**
 * Generates a list of all the tables
 *
 * @return list with all tables
 */
@GET
public static TableInformationList getTables() {
    TableInformationList tableList = new TableInformationList();
    SortedMap<Table.ID, TableInfo> tableStats = new TreeMap<>();
    if (Monitor.getMmi() != null && Monitor.getMmi().tableMap != null)
        for (Map.Entry<String, TableInfo> te : Monitor.getMmi().tableMap.entrySet()) tableStats.put(Table.ID.of(te.getKey()), te.getValue());
    Map<String, Double> compactingByTable = TableInfoUtil.summarizeTableStats(Monitor.getMmi());
    TableManager tableManager = TableManager.getInstance();
    // Add tables to the list
    for (Map.Entry<String, Table.ID> entry : Tables.getNameToIdMap(HdfsZooInstance.getInstance()).entrySet()) {
        String tableName = entry.getKey();
        Table.ID tableId = entry.getValue();
        TableInfo tableInfo = tableStats.get(tableId);
        TableState tableState = tableManager.getTableState(tableId);
        if (null != tableInfo && !tableState.equals(TableState.OFFLINE)) {
            Double holdTime = compactingByTable.get(tableId.canonicalID());
            if (holdTime == null)
                holdTime = 0.;
            tableList.addTable(new TableInformation(tableName, tableId, tableInfo, holdTime, tableState.name()));
        } else {
            tableList.addTable(new TableInformation(tableName, tableId, tableState.name()));
        }
    }
    return tableList;
}

Example 10 with TableInfo

use of org.apache.accumulo.core.master.thrift.TableInfo in project accumulo by apache.

the class GetMasterStats method main.

public static void main(String[] args) throws Exception {
    MasterClientService.Iface client = null;
    MasterMonitorInfo stats = null;
    Instance instance = HdfsZooInstance.getInstance();
    AccumuloServerContext context = new AccumuloServerContext(instance, new ServerConfigurationFactory(instance));
    while (true) {
        try {
            client = MasterClient.getConnectionWithRetry(context);
            stats = client.getMasterStats(Tracer.traceInfo(), context.rpcCreds());
            break;
        } catch (ThriftNotActiveServiceException e) {
            // Let it loop, fetching a new location
            sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        } finally {
            if (client != null)
                MasterClient.close(client);
        }
    }
    out(0, "State: " + stats.state.name());
    out(0, "Goal State: " + stats.goalState.name());
    if (stats.serversShuttingDown != null && stats.serversShuttingDown.size() > 0) {
        out(0, "Servers to shutdown");
        for (String server : stats.serversShuttingDown) {
            out(1, "%s", server);
        }
    }
    out(0, "Unassigned tablets: %d", stats.unassignedTablets);
    if (stats.badTServers != null && stats.badTServers.size() > 0) {
        out(0, "Bad servers");
        for (Entry<String, Byte> entry : stats.badTServers.entrySet()) {
            out(1, "%s: %d", entry.getKey(), (int) entry.getValue());
        }
    }
    out(0, "Dead tablet servers count: %s", stats.deadTabletServers.size());
    for (DeadServer dead : stats.deadTabletServers) {
        out(1, "Dead tablet server: %s", dead.server);
        out(2, "Last report: %s", new SimpleDateFormat().format(new Date(dead.lastStatus)));
        out(2, "Cause: %s", dead.status);
    }
    out(0, "Bulk imports: %s", stats.bulkImports.size());
    for (BulkImportStatus bulk : stats.bulkImports) {
        out(1, "Import directory: %s", bulk.filename);
        out(2, "Bulk state %s", bulk.state);
        out(2, "Bulk start %s", bulk.startTime);
    }
    if (stats.tableMap != null && stats.tableMap.size() > 0) {
        out(0, "Tables");
        for (Entry<String, TableInfo> entry : stats.tableMap.entrySet()) {
            TableInfo v = entry.getValue();
            out(1, "%s", entry.getKey());
            out(2, "Records: %d", v.recs);
            out(2, "Records in Memory: %d", v.recsInMemory);
            out(2, "Tablets: %d", v.tablets);
            out(2, "Online Tablets: %d", v.onlineTablets);
            out(2, "Ingest Rate: %.2f", v.ingestRate);
            out(2, "Query Rate: %.2f", v.queryRate);
        }
    }
    if (stats.tServerInfo != null && stats.tServerInfo.size() > 0) {
        out(0, "Tablet Servers");
        long now = System.currentTimeMillis();
        for (TabletServerStatus server : stats.tServerInfo) {
            TableInfo summary = TableInfoUtil.summarizeTableStats(server);
            out(1, "Name: %s", server.name);
            out(2, "Ingest: %.2f", summary.ingestRate);
            out(2, "Last Contact: %s", server.lastContact);
            out(2, "OS Load Average: %.2f", server.osLoad);
            out(2, "Queries: %.2f", summary.queryRate);
            out(2, "Time Difference: %.1f", ((now - server.lastContact) / 1000.));
            out(2, "Total Records: %d", summary.recs);
            out(2, "Lookups: %d", server.lookups);
            if (server.holdTime > 0)
                out(2, "Hold Time: %d", server.holdTime);
            if (server.tableMap != null && server.tableMap.size() > 0) {
                out(2, "Tables");
                for (Entry<String, TableInfo> status : server.tableMap.entrySet()) {
                    TableInfo info = status.getValue();
                    out(3, "Table: %s", status.getKey());
                    out(4, "Tablets: %d", info.onlineTablets);
                    out(4, "Records: %d", info.recs);
                    out(4, "Records in Memory: %d", info.recsInMemory);
                    out(4, "Ingest: %.2f", info.ingestRate);
                    out(4, "Queries: %.2f", info.queryRate);
                    out(4, "Major Compacting: %d", info.majors == null ? 0 : info.majors.running);
                    out(4, "Queued for Major Compaction: %d", info.majors == null ? 0 : info.majors.queued);
                    out(4, "Minor Compacting: %d", info.minors == null ? 0 : info.minors.running);
                    out(4, "Queued for Minor Compaction: %d", info.minors == null ? 0 : info.minors.queued);
                }
            }
            out(2, "Recoveries: %d", server.logSorts.size());
            for (RecoveryStatus sort : server.logSorts) {
                out(3, "File: %s", sort.name);
                out(3, "Progress: %.2f%%", sort.progress * 100);
                out(3, "Time running: %s", sort.runtime / 1000.);
            }
            out(3, "Bulk imports: %s", stats.bulkImports.size());
            for (BulkImportStatus bulk : stats.bulkImports) {
                out(4, "Import file: %s", bulk.filename);
                out(5, "Bulk state %s", bulk.state);
                out(5, "Bulk start %s", bulk.startTime);
            }
        }
    }
}