Examples with TableId org.apache.accumulo.core.data.TableId used on opensource projects

Example 21 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class Manager method displayUnassigned.

// The number of unassigned tablets that should be assigned: displayed on the monitor page
int displayUnassigned() {
    int result = 0;
    switch(getManagerState()) {
        case NORMAL:
            // Count offline tablets for online tables
            for (TabletGroupWatcher watcher : watchers) {
                TableManager manager = getContext().getTableManager();
                for (Entry<TableId, TableCounts> entry : watcher.getStats().entrySet()) {
                    TableId tableId = entry.getKey();
                    TableCounts counts = entry.getValue();
                    if (manager.getTableState(tableId) == TableState.ONLINE) {
                        result += counts.unassigned() + counts.assignedToDeadServers() + counts.assigned() + counts.suspended();
                    }
                }
            }
            break;
        case SAFE_MODE:
            // Count offline tablets for the metadata table
            for (TabletGroupWatcher watcher : watchers) {
                TableCounts counts = watcher.getStats(MetadataTable.ID);
                result += counts.unassigned() + counts.suspended();
            }
            break;
        case UNLOAD_METADATA_TABLETS:
        case UNLOAD_ROOT_TABLET:
            for (TabletGroupWatcher watcher : watchers) {
                TableCounts counts = watcher.getStats(MetadataTable.ID);
                result += counts.unassigned() + counts.suspended();
            }
            break;
        default:
            break;
    }
    return result;
}

Example 22 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class FinishedWorkUpdater method run.

@Override
public void run() {
    log.trace("Looking for finished replication work");
    if (!ReplicationTable.isOnline(client)) {
        log.trace("Replication table is not yet online, will retry");
        return;
    }
    BatchScanner bs;
    BatchWriter replBw;
    try {
        bs = ReplicationTable.getBatchScanner(client, 4);
        replBw = ReplicationTable.getBatchWriter(client);
    } catch (ReplicationTableOfflineException e) {
        log.debug("Table is no longer online, will retry");
        return;
    }
    IteratorSetting cfg = new IteratorSetting(50, WholeRowIterator.class);
    bs.addScanIterator(cfg);
    WorkSection.limit(bs);
    bs.setRanges(Collections.singleton(new Range()));
    try {
        for (Entry<Key, Value> serializedRow : bs) {
            SortedMap<Key, Value> wholeRow;
            try {
                wholeRow = WholeRowIterator.decodeRow(serializedRow.getKey(), serializedRow.getValue());
            } catch (IOException e) {
                log.warn("Could not deserialize whole row with key {}", serializedRow.getKey().toStringNoTruncate(), e);
                continue;
            }
            log.debug("Processing work progress for {} with {} columns", serializedRow.getKey().getRow(), wholeRow.size());
            Map<TableId, Long> tableIdToProgress = new HashMap<>();
            boolean error = false;
            Text buffer = new Text();
            // We want to determine what the minimum point that all Work entries have replicated to
            for (Entry<Key, Value> entry : wholeRow.entrySet()) {
                Status status;
                try {
                    status = Status.parseFrom(entry.getValue().get());
                } catch (InvalidProtocolBufferException e) {
                    log.warn("Could not deserialize protobuf for {}", entry.getKey(), e);
                    error = true;
                    break;
                }
                // Get the replication target for the work record
                entry.getKey().getColumnQualifier(buffer);
                ReplicationTarget target = ReplicationTarget.from(buffer);
                // Initialize the value in the map if we don't have one
                if (!tableIdToProgress.containsKey(target.getSourceTableId())) {
                    tableIdToProgress.put(target.getSourceTableId(), Long.MAX_VALUE);
                }
                // Find the minimum value for begin (everyone has replicated up to this offset in the
                // file)
                tableIdToProgress.put(target.getSourceTableId(), Math.min(tableIdToProgress.get(target.getSourceTableId()), status.getBegin()));
            }
            if (error) {
                continue;
            }
            // Update the replication table for each source table we found work records for
            for (Entry<TableId, Long> entry : tableIdToProgress.entrySet()) {
                // anything
                if (entry.getValue() == 0) {
                    continue;
                }
                serializedRow.getKey().getRow(buffer);
                log.debug("For {}, source table ID {} has replicated through {}", serializedRow.getKey().getRow(), entry.getKey(), entry.getValue());
                Mutation replMutation = new Mutation(buffer);
                // Set that we replicated at least this much data, ignoring the other fields
                Status updatedStatus = StatusUtil.replicated(entry.getValue());
                Value serializedUpdatedStatus = ProtobufUtil.toValue(updatedStatus);
                // Pull the sourceTableId into a Text
                TableId srcTableId = entry.getKey();
                // Make the mutation
                StatusSection.add(replMutation, srcTableId, serializedUpdatedStatus);
                log.debug("Updating replication status entry for {} with {}", serializedRow.getKey().getRow(), ProtobufUtil.toString(updatedStatus));
                try {
                    replBw.addMutation(replMutation);
                } catch (MutationsRejectedException e) {
                    log.error("Error writing mutations to update replication Status" + " messages in StatusSection, will retry", e);
                    return;
                }
            }
        }
    } finally {
        log.debug("Finished updating files with completed replication work");
        bs.close();
        try {
            replBw.close();
        } catch (MutationsRejectedException e) {
            log.error("Error writing mutations to update replication Status" + " messages in StatusSection, will retry", e);
        }
    }
}

Example 23 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class RemoveCompleteReplicationRecords method removeRowIfNecessary.

protected long removeRowIfNecessary(BatchWriter bw, SortedMap<Key, Value> columns, Text row, Text colf, Text colq) {
    long recordsRemoved = 0;
    if (columns.isEmpty()) {
        return recordsRemoved;
    }
    Mutation m = new Mutation(row);
    Map<TableId, Long> tableToTimeCreated = new HashMap<>();
    for (Entry<Key, Value> entry : columns.entrySet()) {
        Status status = null;
        try {
            status = Status.parseFrom(entry.getValue().get());
        } catch (InvalidProtocolBufferException e) {
            log.error("Encountered unparsable protobuf for key: {}", entry.getKey().toStringNoTruncate());
            continue;
        }
        // If a column in the row isn't ready for removal, we keep the whole row
        if (!StatusUtil.isSafeForRemoval(status)) {
            return 0L;
        }
        Key k = entry.getKey();
        k.getColumnFamily(colf);
        k.getColumnQualifier(colq);
        log.debug("Removing {} {}:{} from replication table", row, colf, colq);
        m.putDelete(colf, colq);
        TableId tableId;
        if (StatusSection.NAME.equals(colf)) {
            tableId = TableId.of(colq.toString());
        } else if (WorkSection.NAME.equals(colf)) {
            ReplicationTarget target = ReplicationTarget.from(colq);
            tableId = target.getSourceTableId();
        } else {
            throw new RuntimeException("Got unexpected column");
        }
        if (status.hasCreatedTime()) {
            Long timeClosed = tableToTimeCreated.get(tableId);
            if (timeClosed == null) {
                tableToTimeCreated.put(tableId, status.getCreatedTime());
            } else if (timeClosed != status.getCreatedTime()) {
                log.warn("Found multiple values for timeClosed for {}: {} and {}", row, timeClosed, status.getCreatedTime());
            }
        }
        recordsRemoved++;
    }
    List<Mutation> mutations = new ArrayList<>();
    mutations.add(m);
    for (Entry<TableId, Long> entry : tableToTimeCreated.entrySet()) {
        log.info("Removing order mutation for table {} at {} for {}", entry.getKey(), entry.getValue(), row);
        Mutation orderMutation = OrderSection.createMutation(row.toString(), entry.getValue());
        orderMutation.putDelete(OrderSection.NAME, new Text(entry.getKey().canonical()));
        mutations.add(orderMutation);
    }
    // or not at all.
    try {
        bw.addMutations(mutations);
        bw.flush();
    } catch (MutationsRejectedException e) {
        log.error("Could not submit mutation to remove columns for {} in replication table", row, e);
        return 0L;
    }
    return recordsRemoved;
}

Example 24 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class SequentialWorkAssigner method cleanupFinishedWork.

/**
 * Iterate over the queued work to remove entries that have been completed.
 */
@Override
protected void cleanupFinishedWork() {
    final Iterator<Entry<String, Map<TableId, String>>> queuedWork = queuedWorkByPeerName.entrySet().iterator();
    final InstanceId instanceId = client.instanceOperations().getInstanceId();
    int elementsRemoved = 0;
    // Check the status of all the work we've queued up
    while (queuedWork.hasNext()) {
        // {peer -> {tableId -> workKey, tableId -> workKey, ... }, peer -> ...}
        Entry<String, Map<TableId, String>> workForPeer = queuedWork.next();
        // TableID to workKey (filename and ReplicationTarget)
        Map<TableId, String> queuedReplication = workForPeer.getValue();
        Iterator<Entry<TableId, String>> iter = queuedReplication.entrySet().iterator();
        // the replication task has finished
        while (iter.hasNext()) {
            // tableID -> workKey
            Entry<TableId, String> entry = iter.next();
            // Null equates to the work for this target was finished
            if (zooCache.get(ZooUtil.getRoot(instanceId) + ReplicationConstants.ZOO_WORK_QUEUE + "/" + entry.getValue()) == null) {
                log.debug("Removing {} from work assignment state", entry.getValue());
                iter.remove();
                elementsRemoved++;
            }
        }
    }
    log.info("Removed {} elements from internal workqueue state because the work was complete", elementsRemoved);
}

Example 25 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class StatusMaker method run.

public void run() {
    Span span = TraceUtil.startSpan(this.getClass(), "replicationStatusMaker");
    try (Scope scope = span.makeCurrent()) {
        // Read from a source table (typically accumulo.metadata)
        final Scanner s;
        try {
            s = client.createScanner(sourceTableName, Authorizations.EMPTY);
        } catch (TableNotFoundException e) {
            throw new RuntimeException(e);
        }
        // Only pull replication records
        s.fetchColumnFamily(ReplicationSection.COLF);
        s.setRange(ReplicationSection.getRange());
        Text file = new Text();
        for (Entry<Key, Value> entry : s) {
            // Get a writer to the replication table
            if (replicationWriter == null) {
                // Ensures table is online
                try {
                    ReplicationTable.setOnline(client);
                    replicationWriter = ReplicationTable.getBatchWriter(client);
                } catch (ReplicationTableOfflineException | AccumuloSecurityException | AccumuloException e) {
                    log.warn("Replication table did not come online");
                    replicationWriter = null;
                    return;
                }
            }
            // Extract the useful bits from the status key
            ReplicationSection.getFile(entry.getKey(), file);
            TableId tableId = ReplicationSection.getTableId(entry.getKey());
            Status status;
            try {
                status = Status.parseFrom(entry.getValue().get());
            } catch (InvalidProtocolBufferException e) {
                log.warn("Could not deserialize protobuf for {}", file);
                continue;
            }
            log.debug("Creating replication status record for {} on table {} with {}.", file, tableId, ProtobufUtil.toString(status));
            Span childSpan = TraceUtil.startSpan(this.getClass(), "createStatusMutations");
            try (Scope childScope = span.makeCurrent()) {
                // Create entries in the replication table from the metadata table
                if (!addStatusRecord(file, tableId, entry.getValue())) {
                    continue;
                }
            } catch (Exception e) {
                TraceUtil.setException(childSpan, e, true);
                throw e;
            } finally {
                childSpan.end();
            }
            if (status.getClosed()) {
                Span closedSpan = TraceUtil.startSpan(this.getClass(), "recordStatusOrder");
                try (Scope childScope = closedSpan.makeCurrent()) {
                    if (!addOrderRecord(file, tableId, status, entry.getValue())) {
                        continue;
                    }
                } catch (Exception e) {
                    TraceUtil.setException(closedSpan, e, true);
                    throw e;
                } finally {
                    closedSpan.end();
                }
                Span deleteSpan = TraceUtil.startSpan(this.getClass(), "deleteClosedStatus");
                try (Scope childScope = deleteSpan.makeCurrent()) {
                    deleteStatusRecord(entry.getKey());
                } catch (Exception e) {
                    TraceUtil.setException(deleteSpan, e, true);
                    throw e;
                } finally {
                    deleteSpan.end();
                }
            }
        }
    } catch (Exception e) {
        TraceUtil.setException(span, e, true);
        throw e;
    } finally {
        span.end();
    }
}