use of org.apache.accumulo.server.log.WalStateManager.WalMarkerException in project accumulo by apache.
the class VolumeIT method verifyVolumesUsed.
private void verifyVolumesUsed(String tableName, boolean shouldExist, Path... paths) throws Exception {
Connector conn = getConnector();
List<String> expected = new ArrayList<>();
for (int i = 0; i < 100; i++) {
String row = String.format("%06d", i * 100 + 3);
expected.add(row + ":cf1:cq1:1");
}
if (!conn.tableOperations().exists(tableName)) {
Assert.assertFalse(shouldExist);
writeData(tableName, conn);
verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
conn.tableOperations().flush(tableName, null, null, true);
}
verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
Table.ID tableId = Table.ID.of(conn.tableOperations().tableIdMap().get(tableName));
try (Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(metaScanner);
metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
metaScanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange());
int[] counts = new int[paths.length];
outer: for (Entry<Key, Value> entry : metaScanner) {
String cf = entry.getKey().getColumnFamily().toString();
String cq = entry.getKey().getColumnQualifier().toString();
String path;
if (cf.equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME.toString()))
path = cq;
else
path = entry.getValue().toString();
for (int i = 0; i < paths.length; i++) {
if (path.startsWith(paths[i].toString())) {
counts[i]++;
continue outer;
}
}
Assert.fail("Unexpected volume " + path);
}
// keep retrying until WAL state information in ZooKeeper stabilizes or until test times out
retry: while (true) {
Instance i = conn.getInstance();
ZooReaderWriter zk = new ZooReaderWriter(i.getZooKeepers(), i.getZooKeepersSessionTimeOut(), "");
WalStateManager wals = new WalStateManager(i, zk);
try {
outer: for (Entry<Path, WalState> entry : wals.getAllState().entrySet()) {
for (Path path : paths) {
if (entry.getKey().toString().startsWith(path.toString())) {
continue outer;
}
}
log.warn("Unexpected volume " + entry.getKey() + " (" + entry.getValue() + ")");
continue retry;
}
} catch (WalMarkerException e) {
Throwable cause = e.getCause();
if (cause instanceof NoNodeException) {
// ignore WALs being cleaned up
continue retry;
}
throw e;
}
break;
}
// if a volume is chosen randomly for each tablet, then the probability that a volume will not be chosen for any tablet is ((num_volumes -
// 1)/num_volumes)^num_tablets. For 100 tablets and 3 volumes the probability that only 2 volumes would be chosen is 2.46e-18
int sum = 0;
for (int count : counts) {
Assert.assertTrue(count > 0);
sum += count;
}
Assert.assertEquals(200, sum);
}
}
use of org.apache.accumulo.server.log.WalStateManager.WalMarkerException in project accumulo by apache.
the class CloseWriteAheadLogReferences method getClosedLogs.
/**
* Construct the set of referenced WALs from zookeeper
*
* @param conn
* Connector
* @return The Set of WALs that are referenced in the metadata table
*/
protected HashSet<String> getClosedLogs(Connector conn) {
WalStateManager wals = new WalStateManager(conn.getInstance(), ZooReaderWriter.getInstance());
HashSet<String> result = new HashSet<>();
try {
for (Entry<Path, WalState> entry : wals.getAllState().entrySet()) {
if (entry.getValue() == WalState.UNREFERENCED || entry.getValue() == WalState.CLOSED) {
Path path = entry.getKey();
log.debug("Found closed WAL " + path.toString());
result.add(path.toString());
}
}
} catch (WalMarkerException e) {
throw new RuntimeException(e);
}
return result;
}
use of org.apache.accumulo.server.log.WalStateManager.WalMarkerException in project accumulo by apache.
the class TabletServer method markUnusedWALs.
private void markUnusedWALs() {
Set<DfsLogger> candidates;
synchronized (closedLogs) {
candidates = new HashSet<>(closedLogs);
}
for (Tablet tablet : getOnlineTablets()) {
candidates.removeAll(tablet.getCurrentLogFiles());
}
try {
TServerInstance session = this.getTabletSession();
for (DfsLogger candidate : candidates) {
log.info("Marking " + candidate.getPath() + " as unreferenced");
walMarker.walUnreferenced(session, candidate.getPath());
}
synchronized (closedLogs) {
closedLogs.removeAll(candidates);
}
} catch (WalMarkerException ex) {
log.info(ex.toString(), ex);
}
}
Aggregations