Examples with MergeStats org.apache.accumulo.manager.state.MergeStats used on opensource projects

Example 1 with MergeStats

use of org.apache.accumulo.manager.state.MergeStats in project accumulo by apache.

the class TabletGroupWatcher method updateMergeState.

private void updateMergeState(Map<TableId, MergeStats> mergeStatsCache) {
    for (MergeStats stats : mergeStatsCache.values()) {
        try {
            MergeState update = stats.nextMergeState(manager.getContext(), manager);
            // not finish
            if (update == MergeState.COMPLETE)
                update = MergeState.NONE;
            if (update != stats.getMergeInfo().getState()) {
                manager.setMergeState(stats.getMergeInfo(), update);
            }
            if (update == MergeState.MERGING) {
                try {
                    if (stats.getMergeInfo().isDelete()) {
                        deleteTablets(stats.getMergeInfo());
                    } else {
                        mergeMetadataRecords(stats.getMergeInfo());
                    }
                    update = MergeState.COMPLETE;
                    manager.setMergeState(stats.getMergeInfo(), update);
                } catch (Exception ex) {
                    Manager.log.error("Unable merge metadata table records", ex);
                }
            }
        } catch (Exception ex) {
            Manager.log.error("Unable to update merge state for merge " + stats.getMergeInfo().getExtent(), ex);
        }
    }
}

Example 2 with MergeStats

use of org.apache.accumulo.manager.state.MergeStats in project accumulo by apache.

the class MergeStateIT method scan.

private MergeStats scan(MockCurrentState state, TabletStateStore metaDataStateStore) {
    MergeStats stats = new MergeStats(state.mergeInfo);
    stats.getMergeInfo().setState(MergeState.WAITING_FOR_OFFLINE);
    for (TabletLocationState tss : metaDataStateStore) {
        stats.update(tss.extent, tss.getState(state.onlineTabletServers()), tss.chopped, false);
    }
    return stats;
}

Example 3 with MergeStats

use of org.apache.accumulo.manager.state.MergeStats in project accumulo by apache.

the class TabletGroupWatcher method run.

@Override
public void run() {
    int[] oldCounts = new int[TabletState.values().length];
    EventCoordinator.Listener eventListener = this.manager.nextEvent.getListener();
    WalStateManager wals = new WalStateManager(manager.getContext());
    while (manager.stillManager()) {
        // slow things down a little, otherwise we spam the logs when there are many wake-up events
        sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        int totalUnloaded = 0;
        int unloaded = 0;
        ClosableIterator<TabletLocationState> iter = null;
        try {
            Map<TableId, MergeStats> mergeStatsCache = new HashMap<>();
            Map<TableId, MergeStats> currentMerges = new HashMap<>();
            for (MergeInfo merge : manager.merges()) {
                if (merge.getExtent() != null) {
                    currentMerges.put(merge.getExtent().tableId(), new MergeStats(merge));
                }
            }
            // Get the current status for the current list of tservers
            SortedMap<TServerInstance, TabletServerStatus> currentTServers = new TreeMap<>();
            for (TServerInstance entry : manager.tserverSet.getCurrentServers()) {
                currentTServers.put(entry, manager.tserverStatus.get(entry));
            }
            if (currentTServers.isEmpty()) {
                eventListener.waitForEvents(Manager.TIME_TO_WAIT_BETWEEN_SCANS);
                synchronized (this) {
                    lastScanServers = Collections.emptySortedSet();
                }
                continue;
            }
            TabletLists tLists = new TabletLists(manager, currentTServers);
            ManagerState managerState = manager.getManagerState();
            int[] counts = new int[TabletState.values().length];
            stats.begin();
            // Walk through the tablets in our store, and work tablets
            // towards their goal
            iter = store.iterator();
            while (iter.hasNext()) {
                TabletLocationState tls = iter.next();
                if (tls == null) {
                    continue;
                }
                // ignore entries for tables that do not exist in zookeeper
                if (manager.getTableManager().getTableState(tls.extent.tableId()) == null)
                    continue;
                // Don't overwhelm the tablet servers with work
                if (tLists.unassigned.size() + unloaded > Manager.MAX_TSERVER_WORK_CHUNK * currentTServers.size()) {
                    flushChanges(tLists, wals);
                    tLists.reset();
                    unloaded = 0;
                    eventListener.waitForEvents(Manager.TIME_TO_WAIT_BETWEEN_SCANS);
                }
                TableId tableId = tls.extent.tableId();
                TableConfiguration tableConf = manager.getContext().getTableConfiguration(tableId);
                MergeStats mergeStats = mergeStatsCache.computeIfAbsent(tableId, k -> {
                    var mStats = currentMerges.get(k);
                    return mStats != null ? mStats : new MergeStats(new MergeInfo());
                });
                TabletGoalState goal = manager.getGoalState(tls, mergeStats.getMergeInfo());
                TServerInstance location = tls.getLocation();
                TabletState state = tls.getState(currentTServers.keySet());
                TabletLogger.missassigned(tls.extent, goal.toString(), state.toString(), tls.future, tls.current, tls.walogs.size());
                stats.update(tableId, state);
                mergeStats.update(tls.extent, state, tls.chopped, !tls.walogs.isEmpty());
                sendChopRequest(mergeStats.getMergeInfo(), state, tls);
                sendSplitRequest(mergeStats.getMergeInfo(), state, tls);
                // Always follow through with assignments
                if (state == TabletState.ASSIGNED) {
                    goal = TabletGoalState.HOSTED;
                }
                // if we are shutting down all the tabletservers, we have to do it in order
                if ((goal == TabletGoalState.SUSPENDED && state == TabletState.HOSTED) && manager.serversToShutdown.equals(currentTServers.keySet())) {
                    if (dependentWatcher != null && dependentWatcher.assignedOrHosted() > 0) {
                        goal = TabletGoalState.HOSTED;
                    }
                }
                if (goal == TabletGoalState.HOSTED) {
                    if ((state != TabletState.HOSTED && !tls.walogs.isEmpty()) && manager.recoveryManager.recoverLogs(tls.extent, tls.walogs))
                        continue;
                    switch(state) {
                        case HOSTED:
                            if (location.equals(manager.migrations.get(tls.extent)))
                                manager.migrations.remove(tls.extent);
                            break;
                        case ASSIGNED_TO_DEAD_SERVER:
                            hostDeadTablet(tLists, tls, location, wals);
                            break;
                        case SUSPENDED:
                            hostSuspendedTablet(tLists, tls, location, tableConf);
                            break;
                        case UNASSIGNED:
                            hostUnassignedTablet(tLists, tls.extent, location);
                            break;
                        case ASSIGNED:
                            // Send another reminder
                            tLists.assigned.add(new Assignment(tls.extent, tls.future));
                            break;
                    }
                } else {
                    switch(state) {
                        case SUSPENDED:
                            // Request a move to UNASSIGNED, so as to allow balancing to continue.
                            tLists.suspendedToGoneServers.add(tls);
                            cancelOfflineTableMigrations(tls.extent);
                            break;
                        case UNASSIGNED:
                            cancelOfflineTableMigrations(tls.extent);
                            break;
                        case ASSIGNED_TO_DEAD_SERVER:
                            unassignDeadTablet(tLists, tls, wals);
                            break;
                        case HOSTED:
                            TServerConnection client = manager.tserverSet.getConnection(location);
                            if (client != null) {
                                client.unloadTablet(manager.managerLock, tls.extent, goal.howUnload(), manager.getSteadyTime());
                                unloaded++;
                                totalUnloaded++;
                            } else {
                                Manager.log.warn("Could not connect to server {}", location);
                            }
                            break;
                        case ASSIGNED:
                            break;
                    }
                }
                counts[state.ordinal()]++;
            }
            flushChanges(tLists, wals);
            // provide stats after flushing changes to avoid race conditions w/ delete table
            stats.end(managerState);
            // Report changes
            for (TabletState state : TabletState.values()) {
                int i = state.ordinal();
                if (counts[i] > 0 && counts[i] != oldCounts[i]) {
                    manager.nextEvent.event("[%s]: %d tablets are %s", store.name(), counts[i], state.name());
                }
            }
            Manager.log.debug(String.format("[%s]: scan time %.2f seconds", store.name(), stats.getScanTime() / 1000.));
            oldCounts = counts;
            if (totalUnloaded > 0) {
                manager.nextEvent.event("[%s]: %d tablets unloaded", store.name(), totalUnloaded);
            }
            updateMergeState(mergeStatsCache);
            synchronized (this) {
                lastScanServers = ImmutableSortedSet.copyOf(currentTServers.keySet());
            }
            if (manager.tserverSet.getCurrentServers().equals(currentTServers.keySet())) {
                Manager.log.debug(String.format("[%s] sleeping for %.2f seconds", store.name(), Manager.TIME_TO_WAIT_BETWEEN_SCANS / 1000.));
                eventListener.waitForEvents(Manager.TIME_TO_WAIT_BETWEEN_SCANS);
            } else {
                Manager.log.info("Detected change in current tserver set, re-running state machine.");
            }
        } catch (Exception ex) {
            Manager.log.error("Error processing table state for store " + store.name(), ex);
            if (ex.getCause() != null && ex.getCause() instanceof BadLocationStateException) {
                repairMetadata(((BadLocationStateException) ex.getCause()).getEncodedEndRow());
            } else {
                sleepUninterruptibly(Manager.WAIT_BETWEEN_ERRORS, TimeUnit.MILLISECONDS);
            }
        } finally {
            if (iter != null) {
                try {
                    iter.close();
                } catch (IOException ex) {
                    Manager.log.warn("Error closing TabletLocationState iterator: " + ex, ex);
                }
            }
        }
    }
}

Example 4 with MergeStats

use of org.apache.accumulo.manager.state.MergeStats in project accumulo by apache.

the class MergeStateIT method test.

@Test
public void test() throws Exception {
    ServerContext context = getServerContext();
    try (AccumuloClient accumuloClient = Accumulo.newClient().from(getClientProperties()).build()) {
        accumuloClient.securityOperations().grantTablePermission(accumuloClient.whoami(), MetadataTable.NAME, TablePermission.WRITE);
        BatchWriter bw = accumuloClient.createBatchWriter(MetadataTable.NAME);
        // Create a fake METADATA table with these splits
        String[] splits = { "a", "e", "j", "o", "t", "z" };
        // create metadata for a table "t" with the splits above
        TableId tableId = TableId.of("t");
        Text pr = null;
        for (String s : splits) {
            Text split = new Text(s);
            Mutation prevRow = TabletColumnFamily.createPrevRowMutation(new KeyExtent(tableId, split, pr));
            prevRow.put(CurrentLocationColumnFamily.NAME, new Text("123456"), new Value("127.0.0.1:1234"));
            ChoppedColumnFamily.CHOPPED_COLUMN.put(prevRow, new Value("junk"));
            bw.addMutation(prevRow);
            pr = split;
        }
        // Add the default tablet
        Mutation defaultTablet = TabletColumnFamily.createPrevRowMutation(new KeyExtent(tableId, null, pr));
        defaultTablet.put(CurrentLocationColumnFamily.NAME, new Text("123456"), new Value("127.0.0.1:1234"));
        bw.addMutation(defaultTablet);
        bw.close();
        // Read out the TabletLocationStates
        MockCurrentState state = new MockCurrentState(new MergeInfo(new KeyExtent(tableId, new Text("p"), new Text("e")), MergeInfo.Operation.MERGE));
        // Verify the tablet state: hosted, and count
        TabletStateStore metaDataStateStore = TabletStateStore.getStoreForLevel(DataLevel.USER, context, state);
        int count = 0;
        for (TabletLocationState tss : metaDataStateStore) {
            if (tss != null)
                count++;
        }
        // the normal case is to skip tablets in a good state
        assertEquals(0, count);
        // Create the hole
        // Split the tablet at one end of the range
        Mutation m = TabletColumnFamily.createPrevRowMutation(new KeyExtent(tableId, new Text("t"), new Text("p")));
        TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value("0.5"));
        TabletColumnFamily.OLD_PREV_ROW_COLUMN.put(m, TabletColumnFamily.encodePrevEndRow(new Text("o")));
        update(accumuloClient, m);
        // do the state check
        MergeStats stats = scan(state, metaDataStateStore);
        MergeState newState = stats.nextMergeState(accumuloClient, state);
        assertEquals(MergeState.WAITING_FOR_OFFLINE, newState);
        // unassign the tablets
        try (BatchDeleter deleter = accumuloClient.createBatchDeleter(MetadataTable.NAME, Authorizations.EMPTY, 1000)) {
            deleter.fetchColumnFamily(CurrentLocationColumnFamily.NAME);
            deleter.setRanges(Collections.singletonList(new Range()));
            deleter.delete();
        }
        // now we should be ready to merge but, we have inconsistent metadata
        stats = scan(state, metaDataStateStore);
        assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState(accumuloClient, state));
        // finish the split
        KeyExtent tablet = new KeyExtent(tableId, new Text("p"), new Text("o"));
        m = TabletColumnFamily.createPrevRowMutation(tablet);
        TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value("0.5"));
        update(accumuloClient, m);
        metaDataStateStore.setLocations(Collections.singletonList(new Assignment(tablet, state.someTServer)));
        // onos... there's a new tablet online
        stats = scan(state, metaDataStateStore);
        assertEquals(MergeState.WAITING_FOR_CHOPPED, stats.nextMergeState(accumuloClient, state));
        // chop it
        m = TabletColumnFamily.createPrevRowMutation(tablet);
        ChoppedColumnFamily.CHOPPED_COLUMN.put(m, new Value("junk"));
        update(accumuloClient, m);
        stats = scan(state, metaDataStateStore);
        assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState(accumuloClient, state));
        // take it offline
        m = TabletColumnFamily.createPrevRowMutation(tablet);
        Collection<Collection<String>> walogs = Collections.emptyList();
        metaDataStateStore.unassign(Collections.singletonList(new TabletLocationState(tablet, null, state.someTServer, null, null, walogs, false)), null);
        // now we can split
        stats = scan(state, metaDataStateStore);
        assertEquals(MergeState.MERGING, stats.nextMergeState(accumuloClient, state));
    }
}