use of org.apache.accumulo.server.zookeeper.ZooCache in project accumulo by apache.
the class MetadataConstraints method check.
@Override
public List<Short> check(Environment env, Mutation mutation) {
ArrayList<Short> violations = null;
Collection<ColumnUpdate> colUpdates = mutation.getUpdates();
// check the row, it should contains at least one ; or end with <
boolean containsSemiC = false;
byte[] row = mutation.getRow();
// always allow rows that fall within reserved areas
if (row.length > 0 && row[0] == '~')
return null;
if (row.length > 2 && row[0] == '!' && row[1] == '!' && row[2] == '~')
return null;
for (byte b : row) {
if (b == ';') {
containsSemiC = true;
}
if (b == ';' || b == '<')
break;
if (!validTableNameChars[0xff & b]) {
violations = addIfNotPresent(violations, 4);
}
}
if (!containsSemiC) {
// see if last row char is <
if (row.length == 0 || row[row.length - 1] != '<') {
violations = addIfNotPresent(violations, 4);
}
} else {
if (row.length == 0) {
violations = addIfNotPresent(violations, 4);
}
}
if (row.length > 0 && row[0] == '!') {
if (row.length < 3 || row[1] != '0' || (row[2] != '<' && row[2] != ';')) {
violations = addIfNotPresent(violations, 4);
}
}
// ensure row is not less than Constants.METADATA_TABLE_ID
if (new Text(row).compareTo(new Text(MetadataTable.ID.getUtf8())) < 0) {
violations = addViolation(violations, 5);
}
boolean checkedBulk = false;
for (ColumnUpdate columnUpdate : colUpdates) {
Text columnFamily = new Text(columnUpdate.getColumnFamily());
if (columnUpdate.isDeleted()) {
if (!isValidColumn(columnUpdate)) {
violations = addViolation(violations, 2);
}
continue;
}
if (columnUpdate.getValue().length == 0 && !columnFamily.equals(ScanFileColumnFamily.NAME)) {
violations = addViolation(violations, 6);
}
if (columnFamily.equals(DataFileColumnFamily.NAME)) {
try {
DataFileValue dfv = new DataFileValue(columnUpdate.getValue());
if (dfv.getSize() < 0 || dfv.getNumEntries() < 0) {
violations = addViolation(violations, 1);
}
} catch (NumberFormatException | ArrayIndexOutOfBoundsException nfe) {
violations = addViolation(violations, 1);
}
} else if (columnFamily.equals(ScanFileColumnFamily.NAME)) {
} else if (columnFamily.equals(TabletsSection.BulkFileColumnFamily.NAME)) {
if (!columnUpdate.isDeleted() && !checkedBulk) {
// splits, which also write the time reference, are allowed to write this reference even when
// the transaction is not running because the other half of the tablet is holding a reference
// to the file.
boolean isSplitMutation = false;
// When a tablet is assigned, it re-writes the metadata. It should probably only update the location information,
// but it writes everything. We allow it to re-write the bulk information if it is setting the location.
// See ACCUMULO-1230.
boolean isLocationMutation = false;
HashSet<Text> dataFiles = new HashSet<>();
HashSet<Text> loadedFiles = new HashSet<>();
String tidString = new String(columnUpdate.getValue(), UTF_8);
int otherTidCount = 0;
for (ColumnUpdate update : mutation.getUpdates()) {
if (new ColumnFQ(update).equals(TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN)) {
isSplitMutation = true;
} else if (new Text(update.getColumnFamily()).equals(TabletsSection.CurrentLocationColumnFamily.NAME)) {
isLocationMutation = true;
} else if (new Text(update.getColumnFamily()).equals(DataFileColumnFamily.NAME)) {
dataFiles.add(new Text(update.getColumnQualifier()));
} else if (new Text(update.getColumnFamily()).equals(TabletsSection.BulkFileColumnFamily.NAME)) {
loadedFiles.add(new Text(update.getColumnQualifier()));
if (!new String(update.getValue(), UTF_8).equals(tidString)) {
otherTidCount++;
}
}
}
if (!isSplitMutation && !isLocationMutation) {
long tid = Long.parseLong(tidString);
try {
if (otherTidCount > 0 || !dataFiles.equals(loadedFiles) || !getArbitrator().transactionAlive(Constants.BULK_ARBITRATOR_TYPE, tid)) {
violations = addViolation(violations, 8);
}
} catch (Exception ex) {
violations = addViolation(violations, 8);
}
}
checkedBulk = true;
}
} else {
if (!isValidColumn(columnUpdate)) {
violations = addViolation(violations, 2);
} else if (new ColumnFQ(columnUpdate).equals(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN) && columnUpdate.getValue().length > 0 && (violations == null || !violations.contains((short) 4))) {
KeyExtent ke = new KeyExtent(new Text(mutation.getRow()), (Text) null);
Text per = KeyExtent.decodePrevEndRow(new Value(columnUpdate.getValue()));
boolean prevEndRowLessThanEndRow = per == null || ke.getEndRow() == null || per.compareTo(ke.getEndRow()) < 0;
if (!prevEndRowLessThanEndRow) {
violations = addViolation(violations, 3);
}
} else if (new ColumnFQ(columnUpdate).equals(TabletsSection.ServerColumnFamily.LOCK_COLUMN)) {
if (zooCache == null) {
zooCache = new ZooCache();
}
if (zooRoot == null) {
zooRoot = ZooUtil.getRoot(HdfsZooInstance.getInstance());
}
boolean lockHeld = false;
String lockId = new String(columnUpdate.getValue(), UTF_8);
try {
lockHeld = ZooLock.isLockHeld(zooCache, new ZooUtil.LockID(zooRoot, lockId));
} catch (Exception e) {
log.debug("Failed to verify lock was held {} {}", lockId, e.getMessage());
}
if (!lockHeld) {
violations = addViolation(violations, 7);
}
}
}
}
if (violations != null) {
log.debug("violating metadata mutation : {}", new String(mutation.getRow(), UTF_8));
for (ColumnUpdate update : mutation.getUpdates()) {
log.debug(" update: {}:{} value {}", new String(update.getColumnFamily(), UTF_8), new String(update.getColumnQualifier(), UTF_8), (update.isDeleted() ? "[delete]" : new String(update.getValue(), UTF_8)));
}
}
return violations;
}
use of org.apache.accumulo.server.zookeeper.ZooCache in project accumulo by apache.
the class DistributedWorkQueueWorkAssigner method assignWork.
@Override
public void assignWork() {
if (null == workQueue) {
initializeWorkQueue(conf);
}
initializeQueuedWork();
if (null == zooCache) {
zooCache = new ZooCache();
}
// Get the maximum number of entries we want to queue work for (or the default)
this.maxQueueSize = conf.getCount(Property.REPLICATION_MAX_WORK_QUEUE);
// Scan over the work records, adding the work to the queue
createWork();
// Keep the state of the work we queued correct
cleanupFinishedWork();
}
use of org.apache.accumulo.server.zookeeper.ZooCache in project accumulo by apache.
the class SequentialWorkAssignerTest method basicZooKeeperCleanup.
@Test
public void basicZooKeeperCleanup() throws Exception {
DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class);
ZooCache zooCache = createMock(ZooCache.class);
Instance inst = createMock(Instance.class);
Map<String, Map<Table.ID, String>> queuedWork = new TreeMap<>();
Map<Table.ID, String> cluster1Work = new TreeMap<>();
// Two files for cluster1, one for table '1' and another for table '2' we havce assigned work for
cluster1Work.put(Table.ID.of("1"), DistributedWorkQueueWorkAssignerHelper.getQueueKey("file1", new ReplicationTarget("cluster1", "1", Table.ID.of("1"))));
cluster1Work.put(Table.ID.of("2"), DistributedWorkQueueWorkAssignerHelper.getQueueKey("file2", new ReplicationTarget("cluster1", "2", Table.ID.of("2"))));
queuedWork.put("cluster1", cluster1Work);
assigner.setConnector(conn);
assigner.setZooCache(zooCache);
assigner.setWorkQueue(workQueue);
assigner.setQueuedWork(queuedWork);
expect(conn.getInstance()).andReturn(inst);
expect(inst.getInstanceID()).andReturn("instance");
// file1 replicated
expect(zooCache.get(ZooUtil.getRoot("instance") + ReplicationConstants.ZOO_WORK_QUEUE + "/" + DistributedWorkQueueWorkAssignerHelper.getQueueKey("file1", new ReplicationTarget("cluster1", "1", Table.ID.of("1"))))).andReturn(null);
// file2 still needs to replicate
expect(zooCache.get(ZooUtil.getRoot("instance") + ReplicationConstants.ZOO_WORK_QUEUE + "/" + DistributedWorkQueueWorkAssignerHelper.getQueueKey("file2", new ReplicationTarget("cluster1", "2", Table.ID.of("2"))))).andReturn(new byte[0]);
replay(workQueue, zooCache, conn, inst);
assigner.cleanupFinishedWork();
verify(workQueue, zooCache, conn, inst);
Assert.assertEquals(1, cluster1Work.size());
Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey("file2", new ReplicationTarget("cluster1", "2", Table.ID.of("2"))), cluster1Work.get(Table.ID.of("2")));
}
use of org.apache.accumulo.server.zookeeper.ZooCache in project accumulo by apache.
the class UnorderedWorkAssignerTest method workNotInZooKeeperIsCleanedUp.
@Test
public void workNotInZooKeeperIsCleanedUp() {
Set<String> queuedWork = new LinkedHashSet<>(Arrays.asList("wal1", "wal2"));
assigner.setQueuedWork(queuedWork);
Instance inst = createMock(Instance.class);
ZooCache cache = createMock(ZooCache.class);
assigner.setZooCache(cache);
expect(conn.getInstance()).andReturn(inst);
expect(inst.getInstanceID()).andReturn("id");
expect(cache.get(Constants.ZROOT + "/id" + ReplicationConstants.ZOO_WORK_QUEUE + "/wal1")).andReturn(null);
expect(cache.get(Constants.ZROOT + "/id" + ReplicationConstants.ZOO_WORK_QUEUE + "/wal2")).andReturn(null);
replay(cache, inst, conn);
assigner.cleanupFinishedWork();
verify(cache, inst, conn);
Assert.assertTrue("Queued work was not emptied", queuedWork.isEmpty());
}
Aggregations