Examples with ReplicationTarget org.apache.accumulo.core.replication.ReplicationTarget used on opensource projects

Example 36 with ReplicationTarget

use of org.apache.accumulo.core.replication.ReplicationTarget in project accumulo by apache.

the class UnorderedWorkAssignerIT method createWorkForFilesNeedingIt.

@Test
public void createWorkForFilesNeedingIt() throws Exception {
    ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1", Table.ID.of("1")), target2 = new ReplicationTarget("cluster1", "table2", Table.ID.of("2"));
    Text serializedTarget1 = target1.toText(), serializedTarget2 = target2.toText();
    String keyTarget1 = target1.getPeerName() + DistributedWorkQueueWorkAssignerHelper.KEY_SEPARATOR + target1.getRemoteIdentifier() + DistributedWorkQueueWorkAssignerHelper.KEY_SEPARATOR + target1.getSourceTableId(), keyTarget2 = target2.getPeerName() + DistributedWorkQueueWorkAssignerHelper.KEY_SEPARATOR + target2.getRemoteIdentifier() + DistributedWorkQueueWorkAssignerHelper.KEY_SEPARATOR + target2.getSourceTableId();
    Status.Builder builder = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(false).setCreatedTime(5l);
    Status status1 = builder.build();
    builder.setCreatedTime(10l);
    Status status2 = builder.build();
    // Create two mutations, both of which need replication work done
    BatchWriter bw = ReplicationTable.getBatchWriter(conn);
    String filename1 = UUID.randomUUID().toString(), filename2 = UUID.randomUUID().toString();
    String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2;
    Mutation m = new Mutation(file1);
    WorkSection.add(m, serializedTarget1, ProtobufUtil.toValue(status1));
    bw.addMutation(m);
    m = OrderSection.createMutation(file1, status1.getCreatedTime());
    OrderSection.add(m, target1.getSourceTableId(), ProtobufUtil.toValue(status1));
    bw.addMutation(m);
    m = new Mutation(file2);
    WorkSection.add(m, serializedTarget2, ProtobufUtil.toValue(status2));
    bw.addMutation(m);
    m = OrderSection.createMutation(file2, status2.getCreatedTime());
    OrderSection.add(m, target2.getSourceTableId(), ProtobufUtil.toValue(status2));
    bw.addMutation(m);
    bw.close();
    DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class);
    HashSet<String> queuedWork = new HashSet<>();
    assigner.setQueuedWork(queuedWork);
    assigner.setWorkQueue(workQueue);
    assigner.setMaxQueueSize(Integer.MAX_VALUE);
    // Make sure we expect the invocations in the order they were created
    String key = filename1 + "|" + keyTarget1;
    workQueue.addWork(key, file1);
    expectLastCall().once();
    key = filename2 + "|" + keyTarget2;
    workQueue.addWork(key, file2);
    expectLastCall().once();
    replay(workQueue);
    assigner.createWork();
    verify(workQueue);
}

Example 37 with ReplicationTarget

use of org.apache.accumulo.core.replication.ReplicationTarget in project accumulo by apache.

the class WorkMakerIT method singleUnitSingleTarget.

@Test
public void singleUnitSingleTarget() throws Exception {
    String table = testName.getMethodName();
    conn.tableOperations().create(table);
    Table.ID tableId = Table.ID.of(conn.tableOperations().tableIdMap().get(table));
    String file = "hdfs://localhost:8020/accumulo/wal/123456-1234-1234-12345678";
    // Create a status record for a file
    long timeCreated = System.currentTimeMillis();
    Mutation m = new Mutation(new Path(file).toString());
    m.put(StatusSection.NAME, new Text(tableId.getUtf8()), StatusUtil.fileCreatedValue(timeCreated));
    BatchWriter bw = ReplicationTable.getBatchWriter(conn);
    bw.addMutation(m);
    bw.flush();
    // Assert that we have one record in the status section
    ReplicationTarget expected;
    try (Scanner s = ReplicationTable.getScanner(conn)) {
        StatusSection.limit(s);
        Assert.assertEquals(1, Iterables.size(s));
        MockWorkMaker workMaker = new MockWorkMaker(conn);
        // Invoke the addWorkRecord method to create a Work record from the Status record earlier
        expected = new ReplicationTarget("remote_cluster_1", "4", tableId);
        workMaker.setBatchWriter(bw);
        workMaker.addWorkRecord(new Text(file), StatusUtil.fileCreatedValue(timeCreated), ImmutableMap.of("remote_cluster_1", "4"), tableId);
    }
    // Scan over just the WorkSection
    try (Scanner s = ReplicationTable.getScanner(conn)) {
        WorkSection.limit(s);
        Entry<Key, Value> workEntry = Iterables.getOnlyElement(s);
        Key workKey = workEntry.getKey();
        ReplicationTarget actual = ReplicationTarget.from(workKey.getColumnQualifier());
        Assert.assertEquals(file, workKey.getRow().toString());
        Assert.assertEquals(WorkSection.NAME, workKey.getColumnFamily());
        Assert.assertEquals(expected, actual);
        Assert.assertEquals(workEntry.getValue(), StatusUtil.fileCreatedValue(timeCreated));
    }
}

Example 38 with ReplicationTarget

use of org.apache.accumulo.core.replication.ReplicationTarget in project accumulo by apache.

the class FinishedWorkUpdaterIT method chooseMinimumBeginOffsetInfiniteEnd.

@Test
public void chooseMinimumBeginOffsetInfiniteEnd() throws Exception {
    conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.READ);
    conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.WRITE);
    ReplicationTable.setOnline(conn);
    String file = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    // @formatter:off
    Status stat1 = Status.newBuilder().setBegin(100).setEnd(1000).setClosed(true).setInfiniteEnd(true).build(), stat2 = Status.newBuilder().setBegin(1).setEnd(1000).setClosed(true).setInfiniteEnd(true).build(), stat3 = Status.newBuilder().setBegin(500).setEnd(1000).setClosed(true).setInfiniteEnd(true).build();
    ReplicationTarget target1 = new ReplicationTarget("peer1", "table1", Table.ID.of("1")), target2 = new ReplicationTarget("peer2", "table2", Table.ID.of("1")), target3 = new ReplicationTarget("peer3", "table3", Table.ID.of("1"));
    // @formatter:on
    // Create a single work record for a file to some peer
    BatchWriter bw = ReplicationTable.getBatchWriter(conn);
    Mutation m = new Mutation(file);
    WorkSection.add(m, target1.toText(), ProtobufUtil.toValue(stat1));
    WorkSection.add(m, target2.toText(), ProtobufUtil.toValue(stat2));
    WorkSection.add(m, target3.toText(), ProtobufUtil.toValue(stat3));
    bw.addMutation(m);
    bw.close();
    updater.run();
    try (Scanner s = ReplicationTable.getScanner(conn)) {
        s.setRange(Range.exact(file));
        StatusSection.limit(s);
        Entry<Key, Value> entry = Iterables.getOnlyElement(s);
        Assert.assertEquals(entry.getKey().getColumnFamily(), StatusSection.NAME);
        Assert.assertEquals(entry.getKey().getColumnQualifier().toString(), target1.getSourceTableId().canonicalID());
        // We should only rely on the correct begin attribute being returned
        Status actual = Status.parseFrom(entry.getValue().get());
        Assert.assertEquals(1, actual.getBegin());
    }
}

Example 39 with ReplicationTarget

use of org.apache.accumulo.core.replication.ReplicationTarget in project accumulo by apache.

the class ReplicationResource method getReplicationInformation.

/**
 * Generates the replication table as a JSON object
 *
 * @return Replication list
 */
@GET
public List<ReplicationInformation> getReplicationInformation() throws AccumuloException, AccumuloSecurityException {
    final Connector conn = Monitor.getContext().getConnector();
    final TableOperations tops = conn.tableOperations();
    final Map<String, String> properties = conn.instanceOperations().getSystemConfiguration();
    final Map<String, String> peers = new HashMap<>();
    final String definedPeersPrefix = Property.REPLICATION_PEERS.getKey();
    final ReplicaSystemFactory replicaSystemFactory = new ReplicaSystemFactory();
    // Get the defined peers and what ReplicaSystem impl they're using
    for (Entry<String, String> property : properties.entrySet()) {
        String key = property.getKey();
        // Filter out cruft that we don't want
        if (key.startsWith(definedPeersPrefix) && !key.startsWith(Property.REPLICATION_PEER_USER.getKey()) && !key.startsWith(Property.REPLICATION_PEER_PASSWORD.getKey())) {
            String peerName = property.getKey().substring(definedPeersPrefix.length());
            ReplicaSystem replica;
            try {
                replica = replicaSystemFactory.get(property.getValue());
            } catch (Exception e) {
                log.warn("Could not instantiate ReplicaSystem for {} with configuration {}", property.getKey(), property.getValue(), e);
                continue;
            }
            peers.put(peerName, replica.getClass().getName());
        }
    }
    final String targetPrefix = Property.TABLE_REPLICATION_TARGET.getKey();
    // The total set of configured targets
    Set<ReplicationTarget> allConfiguredTargets = new HashSet<>();
    // Number of files per target we have to replicate
    Map<ReplicationTarget, Long> targetCounts = new HashMap<>();
    Map<String, Table.ID> tableNameToId = Tables.getNameToIdMap(conn.getInstance());
    Map<Table.ID, String> tableIdToName = invert(tableNameToId);
    for (String table : tops.list()) {
        if (MetadataTable.NAME.equals(table) || RootTable.NAME.equals(table)) {
            continue;
        }
        Table.ID localId = tableNameToId.get(table);
        if (null == localId) {
            log.trace("Could not determine ID for {}", table);
            continue;
        }
        Iterable<Entry<String, String>> propertiesForTable;
        try {
            propertiesForTable = tops.getProperties(table);
        } catch (TableNotFoundException e) {
            log.warn("Could not fetch properties for {}", table, e);
            continue;
        }
        for (Entry<String, String> prop : propertiesForTable) {
            if (prop.getKey().startsWith(targetPrefix)) {
                String peerName = prop.getKey().substring(targetPrefix.length());
                String remoteIdentifier = prop.getValue();
                ReplicationTarget target = new ReplicationTarget(peerName, remoteIdentifier, localId);
                allConfiguredTargets.add(target);
            }
        }
    }
    // Read over the queued work
    BatchScanner bs;
    try {
        bs = conn.createBatchScanner(ReplicationTable.NAME, Authorizations.EMPTY, 4);
    } catch (TableOfflineException | TableNotFoundException e) {
        log.error("Could not read replication table", e);
        return Collections.emptyList();
    }
    bs.setRanges(Collections.singleton(new Range()));
    WorkSection.limit(bs);
    try {
        Text buffer = new Text();
        for (Entry<Key, Value> entry : bs) {
            Key k = entry.getKey();
            k.getColumnQualifier(buffer);
            ReplicationTarget target = ReplicationTarget.from(buffer);
            // TODO ACCUMULO-2835 once explicit lengths are tracked, we can give size-based estimates instead of just file-based
            Long count = targetCounts.get(target);
            if (null == count) {
                targetCounts.put(target, 1l);
            } else {
                targetCounts.put(target, count + 1);
            }
        }
    } finally {
        bs.close();
    }
    List<ReplicationInformation> replicationInformation = new ArrayList<>();
    for (ReplicationTarget configuredTarget : allConfiguredTargets) {
        String tableName = tableIdToName.get(configuredTarget.getSourceTableId());
        if (null == tableName) {
            log.trace("Could not determine table name from id {}", configuredTarget.getSourceTableId());
            continue;
        }
        String replicaSystemClass = peers.get(configuredTarget.getPeerName());
        if (null == replicaSystemClass) {
            log.trace("Could not determine configured ReplicaSystem for {}", configuredTarget.getPeerName());
            continue;
        }
        Long numFiles = targetCounts.get(configuredTarget);
        replicationInformation.add(new ReplicationInformation(tableName, configuredTarget.getPeerName(), configuredTarget.getRemoteIdentifier(), replicaSystemClass, (null == numFiles) ? 0 : numFiles));
    }
    return replicationInformation;
}

Example 40 with ReplicationTarget

use of org.apache.accumulo.core.replication.ReplicationTarget in project accumulo by apache.

the class DistributedWorkQueueWorkAssignerHelperTest method createsValidZKNodeName.

@Test
public void createsValidZKNodeName() {
    Path p = new Path("/accumulo/wals/tserver+port/" + UUID.randomUUID().toString());
    ReplicationTarget target = new ReplicationTarget("cluster1", "table1", Table.ID.of("1"));
    String key = DistributedWorkQueueWorkAssignerHelper.getQueueKey(p.toString(), target);
    PathUtils.validatePath(key);
}