Examples with ReplicaSystem org.apache.accumulo.server.replication.ReplicaSystem used on opensource projects

Example 1 with ReplicaSystem

use of org.apache.accumulo.server.replication.ReplicaSystem in project accumulo by apache.

the class ReplicationProcessor method process.

@Override
public void process(String workID, byte[] data) {
    ReplicationTarget target = DistributedWorkQueueWorkAssignerHelper.fromQueueKey(workID).getValue();
    String file = new String(data, UTF_8);
    log.debug("Received replication work for {} to {}", file, target);
    ReplicaSystem replica;
    try {
        replica = getReplicaSystem(target);
    } catch (Exception e) {
        log.error("Could not instantiate ReplicaSystem for {}, waiting before returning the work", target, e);
        try {
            // TODO configurable
            Thread.sleep(5000);
        } catch (InterruptedException ie) {
            Thread.currentThread().interrupt();
        }
        return;
    }
    Status status;
    try {
        status = getStatus(file, target);
    } catch (ReplicationTableOfflineException | AccumuloException | AccumuloSecurityException e) {
        log.error("Could not look for replication record", e);
        throw new IllegalStateException("Could not look for replication record", e);
    } catch (InvalidProtocolBufferException e) {
        log.error("Could not deserialize Status from Work section for {} and ", file, target);
        throw new RuntimeException("Could not parse Status for work record", e);
    } catch (NoSuchElementException e) {
        log.error("Assigned work for {} to {} but could not find work record", file, target);
        return;
    }
    log.debug("Current status for {} replicating to {}: {}", file, target, ProtobufUtil.toString(status));
    // We don't need to do anything (shouldn't have gotten this work record in the first place)
    if (!StatusUtil.isWorkRequired(status)) {
        log.info("Received work request for {} and {}, but it does not need replication. Ignoring...", file, target);
        return;
    }
    // Sanity check that nothing bad happened and our replication source still exists
    Path filePath = new Path(file);
    try {
        if (!doesFileExist(filePath, target)) {
            return;
        }
    } catch (IOException e) {
        log.error("Could not determine if file exists {}", filePath, e);
        throw new RuntimeException(e);
    }
    log.debug("Replicating {} to {} using {}", filePath, target, replica.getClass().getName());
    Status newStatus = replica.replicate(filePath, status, target, getHelper());
    log.debug("Finished replicating {}. Original status: {}, New status: {}", filePath, status, newStatus);
}

Example 2 with ReplicaSystem

use of org.apache.accumulo.server.replication.ReplicaSystem in project accumulo by apache.

the class ReplicationProcessorTest method filesWhichMakeNoProgressArentReplicatedAgain.

@Test
public void filesWhichMakeNoProgressArentReplicatedAgain() throws Exception {
    ReplicaSystem replica = EasyMock.createMock(ReplicaSystem.class);
    ReplicaSystemHelper helper = EasyMock.createMock(ReplicaSystemHelper.class);
    ReplicationProcessor proc = EasyMock.createMockBuilder(ReplicationProcessor.class).addMockedMethods("getReplicaSystem", "doesFileExist", "getStatus", "getHelper").createMock();
    ReplicationTarget target = new ReplicationTarget("peer", "1", Table.ID.of("1"));
    Status status = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(true).build();
    Path path = new Path("/accumulo");
    String queueKey = DistributedWorkQueueWorkAssignerHelper.getQueueKey(path.toString(), target);
    EasyMock.expect(proc.getReplicaSystem(target)).andReturn(replica);
    EasyMock.expect(proc.getStatus(path.toString(), target)).andReturn(status);
    EasyMock.expect(proc.doesFileExist(path, target)).andReturn(true);
    EasyMock.expect(proc.getHelper()).andReturn(helper);
    EasyMock.expect(replica.replicate(path, status, target, helper)).andReturn(status);
    EasyMock.replay(replica, proc);
    proc.process(queueKey, path.toString().getBytes(UTF_8));
    EasyMock.verify(replica, proc);
}

Example 3 with ReplicaSystem

use of org.apache.accumulo.server.replication.ReplicaSystem in project accumulo by apache.

the class ReplicationResource method getReplicationInformation.

/**
 * Generates the replication table as a JSON object
 *
 * @return Replication list
 */
@GET
public List<ReplicationInformation> getReplicationInformation() throws AccumuloException, AccumuloSecurityException {
    final Connector conn = Monitor.getContext().getConnector();
    final TableOperations tops = conn.tableOperations();
    final Map<String, String> properties = conn.instanceOperations().getSystemConfiguration();
    final Map<String, String> peers = new HashMap<>();
    final String definedPeersPrefix = Property.REPLICATION_PEERS.getKey();
    final ReplicaSystemFactory replicaSystemFactory = new ReplicaSystemFactory();
    // Get the defined peers and what ReplicaSystem impl they're using
    for (Entry<String, String> property : properties.entrySet()) {
        String key = property.getKey();
        // Filter out cruft that we don't want
        if (key.startsWith(definedPeersPrefix) && !key.startsWith(Property.REPLICATION_PEER_USER.getKey()) && !key.startsWith(Property.REPLICATION_PEER_PASSWORD.getKey())) {
            String peerName = property.getKey().substring(definedPeersPrefix.length());
            ReplicaSystem replica;
            try {
                replica = replicaSystemFactory.get(property.getValue());
            } catch (Exception e) {
                log.warn("Could not instantiate ReplicaSystem for {} with configuration {}", property.getKey(), property.getValue(), e);
                continue;
            }
            peers.put(peerName, replica.getClass().getName());
        }
    }
    final String targetPrefix = Property.TABLE_REPLICATION_TARGET.getKey();
    // The total set of configured targets
    Set<ReplicationTarget> allConfiguredTargets = new HashSet<>();
    // Number of files per target we have to replicate
    Map<ReplicationTarget, Long> targetCounts = new HashMap<>();
    Map<String, Table.ID> tableNameToId = Tables.getNameToIdMap(conn.getInstance());
    Map<Table.ID, String> tableIdToName = invert(tableNameToId);
    for (String table : tops.list()) {
        if (MetadataTable.NAME.equals(table) || RootTable.NAME.equals(table)) {
            continue;
        }
        Table.ID localId = tableNameToId.get(table);
        if (null == localId) {
            log.trace("Could not determine ID for {}", table);
            continue;
        }
        Iterable<Entry<String, String>> propertiesForTable;
        try {
            propertiesForTable = tops.getProperties(table);
        } catch (TableNotFoundException e) {
            log.warn("Could not fetch properties for {}", table, e);
            continue;
        }
        for (Entry<String, String> prop : propertiesForTable) {
            if (prop.getKey().startsWith(targetPrefix)) {
                String peerName = prop.getKey().substring(targetPrefix.length());
                String remoteIdentifier = prop.getValue();
                ReplicationTarget target = new ReplicationTarget(peerName, remoteIdentifier, localId);
                allConfiguredTargets.add(target);
            }
        }
    }
    // Read over the queued work
    BatchScanner bs;
    try {
        bs = conn.createBatchScanner(ReplicationTable.NAME, Authorizations.EMPTY, 4);
    } catch (TableOfflineException | TableNotFoundException e) {
        log.error("Could not read replication table", e);
        return Collections.emptyList();
    }
    bs.setRanges(Collections.singleton(new Range()));
    WorkSection.limit(bs);
    try {
        Text buffer = new Text();
        for (Entry<Key, Value> entry : bs) {
            Key k = entry.getKey();
            k.getColumnQualifier(buffer);
            ReplicationTarget target = ReplicationTarget.from(buffer);
            // TODO ACCUMULO-2835 once explicit lengths are tracked, we can give size-based estimates instead of just file-based
            Long count = targetCounts.get(target);
            if (null == count) {
                targetCounts.put(target, 1l);
            } else {
                targetCounts.put(target, count + 1);
            }
        }
    } finally {
        bs.close();
    }
    List<ReplicationInformation> replicationInformation = new ArrayList<>();
    for (ReplicationTarget configuredTarget : allConfiguredTargets) {
        String tableName = tableIdToName.get(configuredTarget.getSourceTableId());
        if (null == tableName) {
            log.trace("Could not determine table name from id {}", configuredTarget.getSourceTableId());
            continue;
        }
        String replicaSystemClass = peers.get(configuredTarget.getPeerName());
        if (null == replicaSystemClass) {
            log.trace("Could not determine configured ReplicaSystem for {}", configuredTarget.getPeerName());
            continue;
        }
        Long numFiles = targetCounts.get(configuredTarget);
        replicationInformation.add(new ReplicationInformation(tableName, configuredTarget.getPeerName(), configuredTarget.getRemoteIdentifier(), replicaSystemClass, (null == numFiles) ? 0 : numFiles));
    }
    return replicationInformation;
}