use of org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException in project accumulo by apache.
the class TabletStateChangeIterator method consume.
@Override
protected void consume() throws IOException {
while (getSource().hasTop()) {
Key k = getSource().getTopKey();
Value v = getSource().getTopValue();
if (onlineTables == null || current == null || masterState != MasterState.NORMAL)
return;
TabletLocationState tls;
try {
tls = MetaDataTableScanner.createTabletLocationState(k, v);
if (tls == null)
return;
} catch (BadLocationStateException e) {
// maybe the master can do something with a tablet with bad/inconsistent state
return;
}
// we always want data about merges
MergeInfo merge = merges.get(tls.extent.getTableId());
if (merge != null) {
// could make this smarter by only returning if the tablet is involved in the merge
return;
}
// always return the information for migrating tablets
if (migrations.contains(tls.extent)) {
return;
}
// is the table supposed to be online or offline?
boolean shouldBeOnline = onlineTables.contains(tls.extent.getTableId());
if (debug) {
log.debug("{} is {} and should be {} line", tls.extent, tls.getState(current), (shouldBeOnline ? "on" : "off"));
}
switch(tls.getState(current)) {
case ASSIGNED:
// we always want data about assigned tablets
return;
case HOSTED:
if (!shouldBeOnline)
return;
break;
case ASSIGNED_TO_DEAD_SERVER:
return;
case SUSPENDED:
case UNASSIGNED:
if (shouldBeOnline)
return;
break;
default:
throw new AssertionError("Inconceivable! The tablet is an unrecognized state: " + tls.getState(current));
}
// table is in the expected state so don't bother returning any information about it
getSource().next();
}
}
use of org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException in project accumulo by apache.
the class TabletGroupWatcher method run.
@Override
public void run() {
Thread.currentThread().setName("Watching " + store.name());
int[] oldCounts = new int[TabletState.values().length];
EventCoordinator.Listener eventListener = this.master.nextEvent.getListener();
WalStateManager wals = new WalStateManager(master.getInstance(), ZooReaderWriter.getInstance());
while (this.master.stillMaster()) {
// slow things down a little, otherwise we spam the logs when there are many wake-up events
sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
masterState = master.getMasterState();
int totalUnloaded = 0;
int unloaded = 0;
ClosableIterator<TabletLocationState> iter = null;
try {
Map<Table.ID, MergeStats> mergeStatsCache = new HashMap<>();
Map<Table.ID, MergeStats> currentMerges = new HashMap<>();
for (MergeInfo merge : master.merges()) {
if (merge.getExtent() != null) {
currentMerges.put(merge.getExtent().getTableId(), new MergeStats(merge));
}
}
// Get the current status for the current list of tservers
SortedMap<TServerInstance, TabletServerStatus> currentTServers = new TreeMap<>();
for (TServerInstance entry : this.master.tserverSet.getCurrentServers()) {
currentTServers.put(entry, this.master.tserverStatus.get(entry));
}
if (currentTServers.size() == 0) {
eventListener.waitForEvents(Master.TIME_TO_WAIT_BETWEEN_SCANS);
synchronized (this) {
lastScanServers = ImmutableSortedSet.of();
}
continue;
}
// Don't move tablets to servers that are shutting down
SortedMap<TServerInstance, TabletServerStatus> destinations = new TreeMap<>(currentTServers);
destinations.keySet().removeAll(this.master.serversToShutdown);
List<Assignment> assignments = new ArrayList<>();
List<Assignment> assigned = new ArrayList<>();
List<TabletLocationState> assignedToDeadServers = new ArrayList<>();
List<TabletLocationState> suspendedToGoneServers = new ArrayList<>();
Map<KeyExtent, TServerInstance> unassigned = new HashMap<>();
Map<TServerInstance, List<Path>> logsForDeadServers = new TreeMap<>();
MasterState masterState = master.getMasterState();
int[] counts = new int[TabletState.values().length];
stats.begin();
// Walk through the tablets in our store, and work tablets
// towards their goal
iter = store.iterator();
while (iter.hasNext()) {
TabletLocationState tls = iter.next();
if (tls == null) {
continue;
}
Master.log.debug("{} location State: {}", store.name(), tls);
// ignore entries for tables that do not exist in zookeeper
if (TableManager.getInstance().getTableState(tls.extent.getTableId()) == null)
continue;
if (Master.log.isTraceEnabled())
Master.log.trace("{} walogs {}", tls, tls.walogs.size());
// Don't overwhelm the tablet servers with work
if (unassigned.size() + unloaded > Master.MAX_TSERVER_WORK_CHUNK * currentTServers.size()) {
flushChanges(destinations, assignments, assigned, assignedToDeadServers, logsForDeadServers, suspendedToGoneServers, unassigned);
assignments.clear();
assigned.clear();
assignedToDeadServers.clear();
suspendedToGoneServers.clear();
unassigned.clear();
unloaded = 0;
eventListener.waitForEvents(Master.TIME_TO_WAIT_BETWEEN_SCANS);
}
Table.ID tableId = tls.extent.getTableId();
TableConfiguration tableConf = this.master.getConfigurationFactory().getTableConfiguration(tableId);
MergeStats mergeStats = mergeStatsCache.get(tableId);
if (mergeStats == null) {
mergeStats = currentMerges.get(tableId);
if (mergeStats == null) {
mergeStats = new MergeStats(new MergeInfo());
}
mergeStatsCache.put(tableId, mergeStats);
}
TabletGoalState goal = this.master.getGoalState(tls, mergeStats.getMergeInfo());
TServerInstance server = tls.getServer();
TabletState state = tls.getState(currentTServers.keySet());
if (Master.log.isTraceEnabled()) {
Master.log.trace("Goal state {} current {} for {}", goal, state, tls.extent);
}
stats.update(tableId, state);
mergeStats.update(tls.extent, state, tls.chopped, !tls.walogs.isEmpty());
sendChopRequest(mergeStats.getMergeInfo(), state, tls);
sendSplitRequest(mergeStats.getMergeInfo(), state, tls);
// Always follow through with assignments
if (state == TabletState.ASSIGNED) {
goal = TabletGoalState.HOSTED;
}
// if we are shutting down all the tabletservers, we have to do it in order
if (goal == TabletGoalState.SUSPENDED && state == TabletState.HOSTED) {
if (this.master.serversToShutdown.equals(currentTServers.keySet())) {
if (dependentWatcher != null && dependentWatcher.assignedOrHosted() > 0) {
goal = TabletGoalState.HOSTED;
}
}
}
if (goal == TabletGoalState.HOSTED) {
if (state != TabletState.HOSTED && !tls.walogs.isEmpty()) {
if (this.master.recoveryManager.recoverLogs(tls.extent, tls.walogs))
continue;
}
switch(state) {
case HOSTED:
if (server.equals(this.master.migrations.get(tls.extent)))
this.master.migrations.remove(tls.extent);
break;
case ASSIGNED_TO_DEAD_SERVER:
assignedToDeadServers.add(tls);
if (server.equals(this.master.migrations.get(tls.extent)))
this.master.migrations.remove(tls.extent);
TServerInstance tserver = tls.futureOrCurrent();
if (!logsForDeadServers.containsKey(tserver)) {
logsForDeadServers.put(tserver, wals.getWalsInUse(tserver));
}
break;
case SUSPENDED:
if (master.getSteadyTime() - tls.suspend.suspensionTime < tableConf.getTimeInMillis(Property.TABLE_SUSPEND_DURATION)) {
// Tablet is suspended. See if its tablet server is back.
TServerInstance returnInstance = null;
Iterator<TServerInstance> find = destinations.tailMap(new TServerInstance(tls.suspend.server, " ")).keySet().iterator();
if (find.hasNext()) {
TServerInstance found = find.next();
if (found.getLocation().equals(tls.suspend.server)) {
returnInstance = found;
}
}
// Old tablet server is back. Return this tablet to its previous owner.
if (returnInstance != null) {
assignments.add(new Assignment(tls.extent, returnInstance));
} else {
// leave suspended, don't ask for a new assignment.
}
} else {
// Treat as unassigned, ask for a new assignment.
unassigned.put(tls.extent, server);
}
break;
case UNASSIGNED:
// maybe it's a finishing migration
TServerInstance dest = this.master.migrations.get(tls.extent);
if (dest != null) {
// if destination is still good, assign it
if (destinations.keySet().contains(dest)) {
assignments.add(new Assignment(tls.extent, dest));
} else {
// get rid of this migration
this.master.migrations.remove(tls.extent);
unassigned.put(tls.extent, server);
}
} else {
unassigned.put(tls.extent, server);
}
break;
case ASSIGNED:
// Send another reminder
assigned.add(new Assignment(tls.extent, tls.future));
break;
}
} else {
switch(state) {
case SUSPENDED:
// Request a move to UNASSIGNED, so as to allow balancing to continue.
suspendedToGoneServers.add(tls);
cancelOfflineTableMigrations(tls);
break;
case UNASSIGNED:
cancelOfflineTableMigrations(tls);
break;
case ASSIGNED_TO_DEAD_SERVER:
assignedToDeadServers.add(tls);
if (!logsForDeadServers.containsKey(tls.futureOrCurrent())) {
logsForDeadServers.put(tls.futureOrCurrent(), wals.getWalsInUse(tls.futureOrCurrent()));
}
break;
case HOSTED:
TServerConnection conn = this.master.tserverSet.getConnection(server);
if (conn != null) {
conn.unloadTablet(this.master.masterLock, tls.extent, goal.howUnload(), master.getSteadyTime());
unloaded++;
totalUnloaded++;
} else {
Master.log.warn("Could not connect to server {}", server);
}
break;
case ASSIGNED:
break;
}
}
counts[state.ordinal()]++;
}
flushChanges(destinations, assignments, assigned, assignedToDeadServers, logsForDeadServers, suspendedToGoneServers, unassigned);
// provide stats after flushing changes to avoid race conditions w/ delete table
stats.end(masterState);
// Report changes
for (TabletState state : TabletState.values()) {
int i = state.ordinal();
if (counts[i] > 0 && counts[i] != oldCounts[i]) {
this.master.nextEvent.event("[%s]: %d tablets are %s", store.name(), counts[i], state.name());
}
}
Master.log.debug(String.format("[%s]: scan time %.2f seconds", store.name(), stats.getScanTime() / 1000.));
oldCounts = counts;
if (totalUnloaded > 0) {
this.master.nextEvent.event("[%s]: %d tablets unloaded", store.name(), totalUnloaded);
}
updateMergeState(mergeStatsCache);
synchronized (this) {
lastScanServers = ImmutableSortedSet.copyOf(currentTServers.keySet());
}
if (this.master.tserverSet.getCurrentServers().equals(currentTServers.keySet())) {
Master.log.debug(String.format("[%s] sleeping for %.2f seconds", store.name(), Master.TIME_TO_WAIT_BETWEEN_SCANS / 1000.));
eventListener.waitForEvents(Master.TIME_TO_WAIT_BETWEEN_SCANS);
} else {
Master.log.info("Detected change in current tserver set, re-running state machine.");
}
} catch (Exception ex) {
Master.log.error("Error processing table state for store " + store.name(), ex);
if (ex.getCause() != null && ex.getCause() instanceof BadLocationStateException) {
repairMetadata(((BadLocationStateException) ex.getCause()).getEncodedEndRow());
} else {
sleepUninterruptibly(Master.WAIT_BETWEEN_ERRORS, TimeUnit.MILLISECONDS);
}
} finally {
if (iter != null) {
try {
iter.close();
} catch (IOException ex) {
Master.log.warn("Error closing TabletLocationState iterator: " + ex, ex);
}
}
}
}
}
use of org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException in project accumulo by apache.
the class MergeStats method verifyMergeConsistency.
private boolean verifyMergeConsistency(Connector connector, CurrentState master) throws TableNotFoundException, IOException {
MergeStats verify = new MergeStats(info);
KeyExtent extent = info.getExtent();
Scanner scanner = connector.createScanner(extent.isMeta() ? RootTable.NAME : MetadataTable.NAME, Authorizations.EMPTY);
MetaDataTableScanner.configureScanner(scanner, master);
Text start = extent.getPrevEndRow();
if (start == null) {
start = new Text();
}
Table.ID tableId = extent.getTableId();
Text first = KeyExtent.getMetadataEntry(tableId, start);
Range range = new Range(first, false, null, true);
scanner.setRange(range.clip(MetadataSchema.TabletsSection.getRange()));
KeyExtent prevExtent = null;
log.debug("Scanning range {}", range);
for (Entry<Key, Value> entry : scanner) {
TabletLocationState tls;
try {
tls = MetaDataTableScanner.createTabletLocationState(entry.getKey(), entry.getValue());
} catch (BadLocationStateException e) {
log.error("{}", e.getMessage(), e);
return false;
}
log.debug("consistency check: {} walogs {}", tls, tls.walogs.size());
if (!tls.extent.getTableId().equals(tableId)) {
break;
}
if (!tls.walogs.isEmpty() && verify.getMergeInfo().needsToBeChopped(tls.extent)) {
log.debug("failing consistency: needs to be chopped {}", tls.extent);
return false;
}
if (prevExtent == null) {
// this is the first tablet observed, it must be offline and its prev row must be less than the start of the merge range
if (tls.extent.getPrevEndRow() != null && tls.extent.getPrevEndRow().compareTo(start) > 0) {
log.debug("failing consistency: prev row is too high {}", start);
return false;
}
if (tls.getState(master.onlineTabletServers()) != TabletState.UNASSIGNED && tls.getState(master.onlineTabletServers()) != TabletState.SUSPENDED) {
log.debug("failing consistency: assigned or hosted {}", tls);
return false;
}
} else if (!tls.extent.isPreviousExtent(prevExtent)) {
log.debug("hole in {}", MetadataTable.NAME);
return false;
}
prevExtent = tls.extent;
verify.update(tls.extent, tls.getState(master.onlineTabletServers()), tls.chopped, !tls.walogs.isEmpty());
// stop when we've seen the tablet just beyond our range
if (tls.extent.getPrevEndRow() != null && extent.getEndRow() != null && tls.extent.getPrevEndRow().compareTo(extent.getEndRow()) > 0) {
break;
}
}
log.debug("chopped {} v.chopped {} unassigned {} v.unassigned {} verify.total {}", chopped, verify.chopped, unassigned, verify.unassigned, verify.total);
return chopped == verify.chopped && unassigned == verify.unassigned && unassigned == verify.total;
}
use of org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException in project accumulo by apache.
the class RootTabletStateStoreTest method testRootTabletStateStore.
@Test
public void testRootTabletStateStore() throws DistributedStoreException {
ZooTabletStateStore tstore = new ZooTabletStateStore(new FakeZooStore());
KeyExtent root = RootTable.EXTENT;
String sessionId = "this is my unique session data";
TServerInstance server = new TServerInstance(HostAndPort.fromParts("127.0.0.1", 10000), sessionId);
List<Assignment> assignments = Collections.singletonList(new Assignment(root, server));
tstore.setFutureLocations(assignments);
int count = 0;
for (TabletLocationState location : tstore) {
assertEquals(location.extent, root);
assertEquals(location.future, server);
assertNull(location.current);
count++;
}
assertEquals(count, 1);
tstore.setLocations(assignments);
count = 0;
for (TabletLocationState location : tstore) {
assertEquals(location.extent, root);
assertNull(location.future);
assertEquals(location.current, server);
count++;
}
assertEquals(count, 1);
TabletLocationState assigned = null;
try {
assigned = new TabletLocationState(root, server, null, null, null, null, false);
} catch (BadLocationStateException e) {
fail("Unexpected error " + e);
}
tstore.unassign(Collections.singletonList(assigned), null);
count = 0;
for (TabletLocationState location : tstore) {
assertEquals(location.extent, root);
assertNull(location.future);
assertNull(location.current);
count++;
}
assertEquals(count, 1);
KeyExtent notRoot = new KeyExtent(Table.ID.of("0"), null, null);
try {
tstore.setLocations(Collections.singletonList(new Assignment(notRoot, server)));
Assert.fail("should not get here");
} catch (IllegalArgumentException ex) {
}
try {
tstore.setFutureLocations(Collections.singletonList(new Assignment(notRoot, server)));
Assert.fail("should not get here");
} catch (IllegalArgumentException ex) {
}
TabletLocationState broken = null;
try {
broken = new TabletLocationState(notRoot, server, null, null, null, null, false);
} catch (BadLocationStateException e) {
fail("Unexpected error " + e);
}
try {
tstore.unassign(Collections.singletonList(broken), null);
Assert.fail("should not get here");
} catch (IllegalArgumentException ex) {
}
}
use of org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException in project accumulo by apache.
the class MetaDataTableScanner method createTabletLocationState.
public static TabletLocationState createTabletLocationState(Key k, Value v) throws IOException, BadLocationStateException {
final SortedMap<Key, Value> decodedRow = WholeRowIterator.decodeRow(k, v);
KeyExtent extent = null;
TServerInstance future = null;
TServerInstance current = null;
TServerInstance last = null;
SuspendingTServer suspend = null;
long lastTimestamp = 0;
List<Collection<String>> walogs = new ArrayList<>();
boolean chopped = false;
for (Entry<Key, Value> entry : decodedRow.entrySet()) {
Key key = entry.getKey();
Text row = key.getRow();
Text cf = key.getColumnFamily();
Text cq = key.getColumnQualifier();
if (cf.compareTo(TabletsSection.FutureLocationColumnFamily.NAME) == 0) {
TServerInstance location = new TServerInstance(entry.getValue(), cq);
if (future != null) {
throw new BadLocationStateException("found two assignments for the same extent " + key.getRow() + ": " + future + " and " + location, entry.getKey().getRow());
}
future = location;
} else if (cf.compareTo(TabletsSection.CurrentLocationColumnFamily.NAME) == 0) {
TServerInstance location = new TServerInstance(entry.getValue(), cq);
if (current != null) {
throw new BadLocationStateException("found two locations for the same extent " + key.getRow() + ": " + current + " and " + location, entry.getKey().getRow());
}
current = location;
} else if (cf.compareTo(LogColumnFamily.NAME) == 0) {
String[] split = entry.getValue().toString().split("\\|")[0].split(";");
walogs.add(Arrays.asList(split));
} else if (cf.compareTo(TabletsSection.LastLocationColumnFamily.NAME) == 0) {
if (lastTimestamp < entry.getKey().getTimestamp())
last = new TServerInstance(entry.getValue(), cq);
} else if (cf.compareTo(ChoppedColumnFamily.NAME) == 0) {
chopped = true;
} else if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.equals(cf, cq)) {
extent = new KeyExtent(row, entry.getValue());
} else if (TabletsSection.SuspendLocationColumn.SUSPEND_COLUMN.equals(cf, cq)) {
suspend = SuspendingTServer.fromValue(entry.getValue());
}
}
if (extent == null) {
String msg = "No prev-row for key extent " + decodedRow;
log.error(msg);
throw new BadLocationStateException(msg, k.getRow());
}
return new TabletLocationState(extent, future, current, last, suspend, walogs, chopped);
}
Aggregations