Examples with Status org.apache.accumulo.server.replication.proto.Replication.Status used on opensource projects

Example 41 with Status

use of org.apache.accumulo.server.replication.proto.Replication.Status in project accumulo by apache.

the class RemoveCompleteReplicationRecords method removeRowIfNecessary.

protected long removeRowIfNecessary(BatchWriter bw, SortedMap<Key, Value> columns, Text row, Text colf, Text colq) {
    long recordsRemoved = 0;
    if (columns.isEmpty()) {
        return recordsRemoved;
    }
    Mutation m = new Mutation(row);
    Map<Table.ID, Long> tableToTimeCreated = new HashMap<>();
    for (Entry<Key, Value> entry : columns.entrySet()) {
        Status status = null;
        try {
            status = Status.parseFrom(entry.getValue().get());
        } catch (InvalidProtocolBufferException e) {
            log.error("Encountered unparsable protobuf for key: {}", entry.getKey().toStringNoTruncate());
            continue;
        }
        // If a column in the row isn't ready for removal, we keep the whole row
        if (!StatusUtil.isSafeForRemoval(status)) {
            return 0l;
        }
        Key k = entry.getKey();
        k.getColumnFamily(colf);
        k.getColumnQualifier(colq);
        log.debug("Removing {} {}:{} from replication table", row, colf, colq);
        m.putDelete(colf, colq);
        Table.ID tableId;
        if (StatusSection.NAME.equals(colf)) {
            tableId = Table.ID.of(colq.toString());
        } else if (WorkSection.NAME.equals(colf)) {
            ReplicationTarget target = ReplicationTarget.from(colq);
            tableId = target.getSourceTableId();
        } else {
            throw new RuntimeException("Got unexpected column");
        }
        if (status.hasCreatedTime()) {
            Long timeClosed = tableToTimeCreated.get(tableId);
            if (null == timeClosed) {
                tableToTimeCreated.put(tableId, status.getCreatedTime());
            } else if (timeClosed != status.getCreatedTime()) {
                log.warn("Found multiple values for timeClosed for {}: {} and {}", row, timeClosed, status.getCreatedTime());
            }
        }
        recordsRemoved++;
    }
    List<Mutation> mutations = new ArrayList<>();
    mutations.add(m);
    for (Entry<Table.ID, Long> entry : tableToTimeCreated.entrySet()) {
        log.info("Removing order mutation for table {} at {} for {}", entry.getKey(), entry.getValue(), row.toString());
        Mutation orderMutation = OrderSection.createMutation(row.toString(), entry.getValue());
        orderMutation.putDelete(OrderSection.NAME, new Text(entry.getKey().getUtf8()));
        mutations.add(orderMutation);
    }
    // or not at all.
    try {
        bw.addMutations(mutations);
        bw.flush();
    } catch (MutationsRejectedException e) {
        log.error("Could not submit mutation to remove columns for {} in replication table", row, e);
        return 0l;
    }
    return recordsRemoved;
}

Example 42 with Status

use of org.apache.accumulo.server.replication.proto.Replication.Status in project accumulo by apache.

the class StatusMaker method run.

public void run() {
    Span span = Trace.start("replicationStatusMaker");
    try {
        // Read from a source table (typically accumulo.metadata)
        final Scanner s;
        try {
            s = conn.createScanner(sourceTableName, Authorizations.EMPTY);
        } catch (TableNotFoundException e) {
            throw new RuntimeException(e);
        }
        // Only pull replication records
        s.fetchColumnFamily(ReplicationSection.COLF);
        s.setRange(ReplicationSection.getRange());
        Text file = new Text();
        for (Entry<Key, Value> entry : s) {
            // Get a writer to the replication table
            if (null == replicationWriter) {
                // Ensures table is online
                try {
                    ReplicationTable.setOnline(conn);
                    replicationWriter = ReplicationTable.getBatchWriter(conn);
                } catch (ReplicationTableOfflineException | AccumuloSecurityException | AccumuloException e) {
                    log.warn("Replication table did not come online");
                    replicationWriter = null;
                    return;
                }
            }
            // Extract the useful bits from the status key
            MetadataSchema.ReplicationSection.getFile(entry.getKey(), file);
            Table.ID tableId = MetadataSchema.ReplicationSection.getTableId(entry.getKey());
            Status status;
            try {
                status = Status.parseFrom(entry.getValue().get());
            } catch (InvalidProtocolBufferException e) {
                log.warn("Could not deserialize protobuf for {}", file);
                continue;
            }
            log.debug("Creating replication status record for {} on table {} with {}.", file, tableId, ProtobufUtil.toString(status));
            Span workSpan = Trace.start("createStatusMutations");
            try {
                // Create entries in the replication table from the metadata table
                if (!addStatusRecord(file, tableId, entry.getValue())) {
                    continue;
                }
            } finally {
                workSpan.stop();
            }
            if (status.getClosed()) {
                Span orderSpan = Trace.start("recordStatusOrder");
                try {
                    if (!addOrderRecord(file, tableId, status, entry.getValue())) {
                        continue;
                    }
                } finally {
                    orderSpan.stop();
                }
                Span deleteSpan = Trace.start("deleteClosedStatus");
                try {
                    deleteStatusRecord(entry.getKey());
                } finally {
                    deleteSpan.stop();
                }
            }
        }
    } finally {
        span.stop();
    }
}

Example 43 with Status

use of org.apache.accumulo.server.replication.proto.Replication.Status in project accumulo by apache.

the class WorkMaker method run.

public void run() {
    if (!ReplicationTable.isOnline(conn)) {
        log.debug("Replication table is not yet online");
        return;
    }
    Span span = Trace.start("replicationWorkMaker");
    try {
        final Scanner s;
        try {
            s = ReplicationTable.getScanner(conn);
            if (null == writer) {
                setBatchWriter(ReplicationTable.getBatchWriter(conn));
            }
        } catch (ReplicationTableOfflineException e) {
            log.warn("Replication table was online, but not anymore");
            writer = null;
            return;
        }
        // Only pull records about data that has been ingested and is ready for replication
        StatusSection.limit(s);
        TableConfiguration tableConf;
        Text file = new Text();
        for (Entry<Key, Value> entry : s) {
            // Extract the useful bits from the status key
            ReplicationSchema.StatusSection.getFile(entry.getKey(), file);
            Table.ID tableId = ReplicationSchema.StatusSection.getTableId(entry.getKey());
            log.debug("Processing replication status record for {} on table {}", file, tableId);
            Status status;
            try {
                status = Status.parseFrom(entry.getValue().get());
            } catch (InvalidProtocolBufferException e) {
                log.error("Could not parse protobuf for {} from table {}", file, tableId);
                continue;
            }
            // TODO put this into a filter on serverside
            if (!shouldCreateWork(status)) {
                log.debug("Not creating work: {}", status.toString());
                continue;
            }
            // Get the table configuration for the table specified by the status record
            tableConf = context.getServerConfigurationFactory().getTableConfiguration(tableId);
            // getTableConfiguration(String) returns null if the table no longer exists
            if (null == tableConf) {
                continue;
            }
            // Pull the relevant replication targets
            // TODO Cache this instead of pulling it every time
            Map<String, String> replicationTargets = getReplicationTargets(tableConf);
            // -- Another scanner over the WorkSection can make this relatively cheap
            if (!replicationTargets.isEmpty()) {
                Span workSpan = Trace.start("createWorkMutations");
                try {
                    addWorkRecord(file, entry.getValue(), replicationTargets, tableId);
                } finally {
                    workSpan.stop();
                }
            } else {
                log.warn("No configured targets for table with ID {}", tableId);
            }
        }
    } finally {
        span.stop();
    }
}

Example 44 with Status

use of org.apache.accumulo.server.replication.proto.Replication.Status in project accumulo by apache.

the class ReplicationUtil method getProgress.

/**
 * Compute a progress string for the replication of the given WAL
 *
 * @param conn
 *          Accumulo Connector
 * @param path
 *          Absolute path to a WAL, or null
 * @param target
 *          ReplicationTarget the WAL is being replicated to
 * @return A status message for a file being replicated
 */
public String getProgress(Connector conn, String path, ReplicationTarget target) {
    // We could try to grep over the table, but without knowing the full file path, we
    // can't find the status quickly
    String status = "Unknown";
    if (null != path) {
        Scanner s;
        try {
            s = ReplicationTable.getScanner(conn);
        } catch (ReplicationTableOfflineException e) {
            log.debug("Replication table no longer online", e);
            return status;
        }
        s.setRange(Range.exact(path));
        s.fetchColumn(WorkSection.NAME, target.toText());
        // Fetch the work entry for this item
        Entry<Key, Value> kv = null;
        try {
            kv = Iterables.getOnlyElement(s);
        } catch (NoSuchElementException e) {
            log.trace("Could not find status of {} replicating to {}", path, target);
            status = "Unknown";
        } finally {
            s.close();
        }
        // If we found the work entry for it, try to compute some progress
        if (null != kv) {
            try {
                Status stat = Status.parseFrom(kv.getValue().get());
                if (StatusUtil.isFullyReplicated(stat)) {
                    status = "Finished";
                } else {
                    if (stat.getInfiniteEnd()) {
                        status = stat.getBegin() + "/&infin; records";
                    } else {
                        status = stat.getBegin() + "/" + stat.getEnd() + " records";
                    }
                }
            } catch (InvalidProtocolBufferException e) {
                log.warn("Could not deserialize protobuf for {}", kv.getKey(), e);
                status = "Unknown";
            }
        }
    }
    return status;
}

Example 45 with Status

use of org.apache.accumulo.server.replication.proto.Replication.Status in project accumulo by apache.

the class StatusFormatter method next.

@Override
public String next() {
    Entry<Key, Value> entry = iterator.next();
    DateFormat timestampFormat = config.willPrintTimestamps() ? config.getDateFormatSupplier().get() : null;
    // If we expected this to be a protobuf, try to parse it, adding a message when it fails to parse
    if (REPLICATION_COLFAMS.contains(entry.getKey().getColumnFamily())) {
        Status status;
        try {
            status = Status.parseFrom(entry.getValue().get());
        } catch (InvalidProtocolBufferException e) {
            log.trace("Could not deserialize protocol buffer for {}", entry.getKey(), e);
            status = null;
        }
        return formatEntry(entry.getKey(), status, timestampFormat);
    } else {
        // Just do the normal thing
        return DefaultFormatter.formatEntry(entry, timestampFormat);
    }
}