Examples with MapCounter org.apache.accumulo.core.util.MapCounter used on opensource projects

Example 1 with MapCounter

use of org.apache.accumulo.core.util.MapCounter in project accumulo by apache.

the class GroupBalancer method balance.

@Override
public long balance(SortedMap<TServerInstance, TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
    if (!shouldBalance(current, migrations)) {
        return 5000;
    }
    if (System.currentTimeMillis() - lastRun < getWaitTime()) {
        return 5000;
    }
    MapCounter<String> groupCounts = new MapCounter<>();
    Map<TServerInstance, TserverGroupInfo> tservers = new HashMap<>();
    for (TServerInstance tsi : current.keySet()) {
        tservers.put(tsi, new TserverGroupInfo(tsi));
    }
    Function<KeyExtent, String> partitioner = getPartitioner();
    // collect stats about current state
    for (Pair<KeyExtent, Location> entry : getLocationProvider()) {
        String group = partitioner.apply(entry.getFirst());
        Location loc = entry.getSecond();
        if (loc.equals(Location.NONE) || !tservers.containsKey(loc.getTserverInstance())) {
            return 5000;
        }
        groupCounts.increment(group, 1);
        TserverGroupInfo tgi = tservers.get(loc.getTserverInstance());
        tgi.addGroup(group);
    }
    Map<String, Integer> expectedCounts = new HashMap<>();
    int totalExtra = 0;
    for (String group : groupCounts.keySet()) {
        long groupCount = groupCounts.get(group);
        totalExtra += groupCount % current.size();
        expectedCounts.put(group, (int) (groupCount / current.size()));
    }
    // The number of extra tablets from all groups that each tserver must have.
    int expectedExtra = totalExtra / current.size();
    int maxExtraGroups = expectedExtra + 1;
    expectedCounts = Collections.unmodifiableMap(expectedCounts);
    tservers = Collections.unmodifiableMap(tservers);
    for (TserverGroupInfo tgi : tservers.values()) {
        tgi.finishedAdding(expectedCounts);
    }
    Moves moves = new Moves();
    // The order of the following steps is important, because as ordered each step should not move any tablets moved by a previous step.
    balanceExpected(tservers, moves);
    if (moves.size() < getMaxMigrations()) {
        balanceExtraExpected(tservers, expectedExtra, moves);
        if (moves.size() < getMaxMigrations()) {
            boolean cont = balanceExtraMultiple(tservers, maxExtraGroups, moves);
            if (cont && moves.size() < getMaxMigrations()) {
                balanceExtraExtra(tservers, maxExtraGroups, moves);
            }
        }
    }
    populateMigrations(tservers.keySet(), migrationsOut, moves);
    lastRun = System.currentTimeMillis();
    return 5000;
}

Example 2 with MapCounter

use of org.apache.accumulo.core.util.MapCounter in project accumulo by apache.

the class TableOperationsImpl method waitForTableStateTransition.

private void waitForTableStateTransition(Table.ID tableId, TableState expectedState) throws AccumuloException, TableNotFoundException, AccumuloSecurityException {
    Text startRow = null;
    Text lastRow = null;
    while (true) {
        if (Tables.getTableState(context.getInstance(), tableId) != expectedState) {
            Tables.clearCache(context.getInstance());
            TableState currentState = Tables.getTableState(context.getInstance(), tableId);
            if (currentState != expectedState) {
                if (!Tables.exists(context.getInstance(), tableId))
                    throw new TableDeletedException(tableId.canonicalID());
                if (currentState == TableState.DELETING)
                    throw new TableNotFoundException(tableId.canonicalID(), "", "Table is being deleted.");
                throw new AccumuloException("Unexpected table state " + tableId + " " + Tables.getTableState(context.getInstance(), tableId) + " != " + expectedState);
            }
        }
        Range range;
        if (startRow == null || lastRow == null)
            range = new KeyExtent(tableId, null, null).toMetadataRange();
        else
            range = new Range(startRow, lastRow);
        String metaTable = MetadataTable.NAME;
        if (tableId.equals(MetadataTable.ID))
            metaTable = RootTable.NAME;
        Scanner scanner = createMetadataScanner(metaTable, range);
        RowIterator rowIter = new RowIterator(scanner);
        KeyExtent lastExtent = null;
        int total = 0;
        int waitFor = 0;
        int holes = 0;
        Text continueRow = null;
        MapCounter<String> serverCounts = new MapCounter<>();
        while (rowIter.hasNext()) {
            Iterator<Entry<Key, Value>> row = rowIter.next();
            total++;
            KeyExtent extent = null;
            String future = null;
            String current = null;
            while (row.hasNext()) {
                Entry<Key, Value> entry = row.next();
                Key key = entry.getKey();
                if (key.getColumnFamily().equals(TabletsSection.FutureLocationColumnFamily.NAME))
                    future = entry.getValue().toString();
                if (key.getColumnFamily().equals(TabletsSection.CurrentLocationColumnFamily.NAME))
                    current = entry.getValue().toString();
                if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key))
                    extent = new KeyExtent(key.getRow(), entry.getValue());
            }
            if ((expectedState == TableState.ONLINE && current == null) || (expectedState == TableState.OFFLINE && (future != null || current != null))) {
                if (continueRow == null)
                    continueRow = extent.getMetadataEntry();
                waitFor++;
                lastRow = extent.getMetadataEntry();
                if (current != null)
                    serverCounts.increment(current, 1);
                if (future != null)
                    serverCounts.increment(future, 1);
            }
            if (!extent.getTableId().equals(tableId)) {
                throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
            }
            if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
                holes++;
            }
            lastExtent = extent;
        }
        if (continueRow != null) {
            startRow = continueRow;
        }
        if (holes > 0 || total == 0) {
            startRow = null;
            lastRow = null;
        }
        if (waitFor > 0 || holes > 0 || total == 0) {
            long waitTime;
            long maxPerServer = 0;
            if (serverCounts.size() > 0) {
                maxPerServer = Collections.max(serverCounts.values());
                waitTime = maxPerServer * 10;
            } else
                waitTime = waitFor * 10;
            waitTime = Math.max(100, waitTime);
            waitTime = Math.min(5000, waitTime);
            log.trace("Waiting for {}({}) tablets, startRow = {} lastRow = {}, holes={} sleeping:{}ms", waitFor, maxPerServer, startRow, lastRow, holes, waitTime);
            sleepUninterruptibly(waitTime, TimeUnit.MILLISECONDS);
        } else {
            break;
        }
    }
}

Example 3 with MapCounter

use of org.apache.accumulo.core.util.MapCounter in project accumulo by apache.

the class SessionManager method getActiveScansPerTable.

public Map<Table.ID, MapCounter<ScanRunState>> getActiveScansPerTable() {
    Map<Table.ID, MapCounter<ScanRunState>> counts = new HashMap<>();
    Set<Entry<Long, Session>> copiedIdleSessions = new HashSet<>();
    synchronized (idleSessions) {
        /**
         * Add sessions so that get the list returned in the active scans call
         */
        for (Session session : idleSessions) {
            copiedIdleSessions.add(Maps.immutableEntry(expiredSessionMarker, session));
        }
    }
    for (Entry<Long, Session> entry : Iterables.concat(sessions.entrySet(), copiedIdleSessions)) {
        Session session = entry.getValue();
        @SuppressWarnings("rawtypes") ScanTask nbt = null;
        Table.ID tableID = null;
        if (session instanceof ScanSession) {
            ScanSession ss = (ScanSession) session;
            nbt = ss.nextBatchTask;
            tableID = ss.extent.getTableId();
        } else if (session instanceof MultiScanSession) {
            MultiScanSession mss = (MultiScanSession) session;
            nbt = mss.lookupTask;
            tableID = mss.threadPoolExtent.getTableId();
        }
        if (nbt == null)
            continue;
        ScanRunState srs = nbt.getScanRunState();
        if (srs == ScanRunState.FINISHED)
            continue;
        MapCounter<ScanRunState> stateCounts = counts.get(tableID);
        if (stateCounts == null) {
            stateCounts = new MapCounter<>();
            counts.put(tableID, stateCounts);
        }
        stateCounts.increment(srs, 1);
    }
    return counts;
}

Example 4 with MapCounter

use of org.apache.accumulo.core.util.MapCounter in project accumulo by apache.

the class CompactionDriver method isReady.

@Override
public long isReady(long tid, Master master) throws Exception {
    String zCancelID = Constants.ZROOT + "/" + master.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_CANCEL_ID;
    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
    if (Long.parseLong(new String(zoo.getData(zCancelID, null))) >= compactId) {
        // compaction was canceled
        throw new AcceptableThriftTableOperationException(tableId.canonicalID(), null, TableOperation.COMPACT, TableOperationExceptionType.OTHER, "Compaction canceled");
    }
    MapCounter<TServerInstance> serversToFlush = new MapCounter<>();
    Connector conn = master.getConnector();
    Scanner scanner;
    if (tableId.equals(MetadataTable.ID)) {
        scanner = new IsolatedScanner(conn.createScanner(RootTable.NAME, Authorizations.EMPTY));
        scanner.setRange(MetadataSchema.TabletsSection.getRange());
    } else {
        scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
        Range range = new KeyExtent(tableId, null, startRow == null ? null : new Text(startRow)).toMetadataRange();
        scanner.setRange(range);
    }
    TabletsSection.ServerColumnFamily.COMPACT_COLUMN.fetch(scanner);
    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
    scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
    long t1 = System.currentTimeMillis();
    RowIterator ri = new RowIterator(scanner);
    int tabletsToWaitFor = 0;
    int tabletCount = 0;
    while (ri.hasNext()) {
        Iterator<Entry<Key, Value>> row = ri.next();
        long tabletCompactID = -1;
        TServerInstance server = null;
        Entry<Key, Value> entry = null;
        while (row.hasNext()) {
            entry = row.next();
            Key key = entry.getKey();
            if (TabletsSection.ServerColumnFamily.COMPACT_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
                tabletCompactID = Long.parseLong(entry.getValue().toString());
            if (TabletsSection.CurrentLocationColumnFamily.NAME.equals(key.getColumnFamily()))
                server = new TServerInstance(entry.getValue(), key.getColumnQualifier());
        }
        if (tabletCompactID < compactId) {
            tabletsToWaitFor++;
            if (server != null)
                serversToFlush.increment(server, 1);
        }
        tabletCount++;
        Text tabletEndRow = new KeyExtent(entry.getKey().getRow(), (Text) null).getEndRow();
        if (tabletEndRow == null || (endRow != null && tabletEndRow.compareTo(new Text(endRow)) >= 0))
            break;
    }
    long scanTime = System.currentTimeMillis() - t1;
    Instance instance = master.getInstance();
    Tables.clearCache(instance);
    if (tabletCount == 0 && !Tables.exists(instance, tableId))
        throw new AcceptableThriftTableOperationException(tableId.canonicalID(), null, TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null);
    if (serversToFlush.size() == 0 && Tables.getTableState(instance, tableId) == TableState.OFFLINE)
        throw new AcceptableThriftTableOperationException(tableId.canonicalID(), null, TableOperation.COMPACT, TableOperationExceptionType.OFFLINE, null);
    if (tabletsToWaitFor == 0)
        return 0;
    for (TServerInstance tsi : serversToFlush.keySet()) {
        try {
            final TServerConnection server = master.getConnection(tsi);
            if (server != null)
                server.compact(master.getMasterLock(), tableId.canonicalID(), startRow, endRow);
        } catch (TException ex) {
            LoggerFactory.getLogger(CompactionDriver.class).error(ex.toString());
        }
    }
    long sleepTime = 500;
    if (serversToFlush.size() > 0)
        // make wait time depend on the server with the most to
        sleepTime = Collections.max(serversToFlush.values()) * sleepTime;
    // compact
    sleepTime = Math.max(2 * scanTime, sleepTime);
    sleepTime = Math.min(sleepTime, 30000);
    return sleepTime;
}

Example 5 with MapCounter

use of org.apache.accumulo.core.util.MapCounter in project accumulo by apache.

the class TabletServer method getStats.

public TabletServerStatus getStats(Map<Table.ID, MapCounter<ScanRunState>> scanCounts) {
    long start = System.currentTimeMillis();
    TabletServerStatus result = new TabletServerStatus();
    Map<KeyExtent, Tablet> onlineTabletsCopy;
    synchronized (this.onlineTablets) {
        onlineTabletsCopy = new HashMap<>(this.onlineTablets);
    }
    Map<String, TableInfo> tables = new HashMap<>();
    for (Entry<KeyExtent, Tablet> entry : onlineTabletsCopy.entrySet()) {
        String tableId = entry.getKey().getTableId().canonicalID();
        TableInfo table = tables.get(tableId);
        if (table == null) {
            table = new TableInfo();
            table.minors = new Compacting();
            table.majors = new Compacting();
            tables.put(tableId, table);
        }
        Tablet tablet = entry.getValue();
        long recs = tablet.getNumEntries();
        table.tablets++;
        table.onlineTablets++;
        table.recs += recs;
        table.queryRate += tablet.queryRate();
        table.queryByteRate += tablet.queryByteRate();
        table.ingestRate += tablet.ingestRate();
        table.ingestByteRate += tablet.ingestByteRate();
        table.scanRate += tablet.scanRate();
        long recsInMemory = tablet.getNumEntriesInMemory();
        table.recsInMemory += recsInMemory;
        if (tablet.isMinorCompactionRunning())
            table.minors.running++;
        if (tablet.isMinorCompactionQueued())
            table.minors.queued++;
        if (tablet.isMajorCompactionRunning())
            table.majors.running++;
        if (tablet.isMajorCompactionQueued())
            table.majors.queued++;
    }
    for (Entry<Table.ID, MapCounter<ScanRunState>> entry : scanCounts.entrySet()) {
        TableInfo table = tables.get(entry.getKey().canonicalID());
        if (table == null) {
            table = new TableInfo();
            tables.put(entry.getKey().canonicalID(), table);
        }
        if (table.scans == null)
            table.scans = new Compacting();
        table.scans.queued += entry.getValue().get(ScanRunState.QUEUED);
        table.scans.running += entry.getValue().get(ScanRunState.RUNNING);
    }
    ArrayList<KeyExtent> offlineTabletsCopy = new ArrayList<>();
    synchronized (this.unopenedTablets) {
        synchronized (this.openingTablets) {
            offlineTabletsCopy.addAll(this.unopenedTablets);
            offlineTabletsCopy.addAll(this.openingTablets);
        }
    }
    for (KeyExtent extent : offlineTabletsCopy) {
        String tableId = extent.getTableId().canonicalID();
        TableInfo table = tables.get(tableId);
        if (table == null) {
            table = new TableInfo();
            tables.put(tableId, table);
        }
        table.tablets++;
    }
    result.lastContact = RelativeTime.currentTimeMillis();
    result.tableMap = tables;
    result.osLoad = ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();
    result.name = getClientAddressString();
    result.holdTime = resourceManager.holdTime();
    result.lookups = seekCount.get();
    result.indexCacheHits = resourceManager.getIndexCache().getStats().hitCount();
    result.indexCacheRequest = resourceManager.getIndexCache().getStats().requestCount();
    result.dataCacheHits = resourceManager.getDataCache().getStats().hitCount();
    result.dataCacheRequest = resourceManager.getDataCache().getStats().requestCount();
    result.logSorts = logSorter.getLogSorts();
    result.flushs = flushCounter.get();
    result.syncs = syncCounter.get();
    result.bulkImports = new ArrayList<>();
    result.bulkImports.addAll(clientHandler.getBulkLoadStatus());
    result.bulkImports.addAll(bulkImportStatus.getBulkLoadStatus());
    result.version = getVersion();
    result.responseTime = System.currentTimeMillis() - start;
    return result;
}