Examples with AccumuloClient org.apache.accumulo.core.client.AccumuloClient used on opensource projects

Example 56 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class ReplicationIT method correctClusterNameInWorkEntry.

@Test
public void correctClusterNameInWorkEntry() throws Exception {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        String table1 = "table1";
        // replication shouldn't be online when we begin
        assertFalse(ReplicationTable.isOnline(client));
        // Create two tables
        client.tableOperations().create(table1);
        int attempts = 5;
        while (attempts > 0) {
            try {
                // Enable replication on table1
                client.tableOperations().setProperty(table1, Property.TABLE_REPLICATION.getKey(), "true");
                // Replicate table1 to cluster1 in the table with id of '4'
                client.tableOperations().setProperty(table1, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "4");
                attempts = 0;
            } catch (Exception e) {
                attempts--;
                if (attempts <= 0) {
                    throw e;
                }
                sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
            }
        }
        // Write some data to table1
        writeSomeData(client, table1, 2000, 50);
        client.tableOperations().flush(table1, null, null, true);
        TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(table1));
        assertNotNull("Table ID was null", tableId);
        // Make sure the replication table exists at this point
        while (!ReplicationTable.isOnline(client)) {
            sleepUninterruptibly(MILLIS_BETWEEN_REPLICATION_TABLE_ONLINE_CHECKS, TimeUnit.MILLISECONDS);
        }
        assertTrue("Replication table did not exist", ReplicationTable.isOnline(client));
        for (int i = 0; i < 5 && !client.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ); i++) {
            Thread.sleep(1000);
        }
        assertTrue(client.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ));
        boolean notFound = true;
        for (int i = 0; i < 10 && notFound; i++) {
            try (Scanner s = ReplicationTable.getScanner(client)) {
                WorkSection.limit(s);
                try {
                    Entry<Key, Value> e = Iterables.getOnlyElement(s);
                    Text expectedColqual = new ReplicationTarget("cluster1", "4", tableId).toText();
                    assertEquals(expectedColqual, e.getKey().getColumnQualifier());
                    notFound = false;
                } catch (NoSuchElementException e) {
                } catch (IllegalArgumentException e) {
                    try (Scanner s2 = ReplicationTable.getScanner(client)) {
                        for (Entry<Key, Value> content : s2) {
                            log.info("{} => {}", content.getKey().toStringNoTruncate(), content.getValue());
                        }
                        fail("Found more than one work section entry");
                    }
                }
                Thread.sleep(500);
            }
        }
        if (notFound) {
            try (Scanner s = ReplicationTable.getScanner(client)) {
                for (Entry<Key, Value> content : s) {
                    log.info("{} => {}", content.getKey().toStringNoTruncate(), content.getValue());
                }
                assertFalse("Did not find the work entry for the status entry", notFound);
            }
        }
    }
}

Example 57 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class ReplicationIT method singleTableWithSingleTarget.

@Test
public void singleTableWithSingleTarget() throws Exception {
    // We want to kill the GC so it doesn't come along and close Status records and mess up the
    // comparisons
    // against expected Status messages.
    getCluster().getClusterControl().stop(ServerType.GARBAGE_COLLECTOR);
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        String table1 = "table1";
        // replication shouldn't be online when we begin
        assertFalse(ReplicationTable.isOnline(client));
        // Create a table
        client.tableOperations().create(table1);
        int attempts = 10;
        // Might think the table doesn't yet exist, retry
        while (attempts > 0) {
            try {
                // Enable replication on table1
                client.tableOperations().setProperty(table1, Property.TABLE_REPLICATION.getKey(), "true");
                // Replicate table1 to cluster1 in the table with id of '4'
                client.tableOperations().setProperty(table1, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "4");
                // Sleep for 100 seconds before saying something is replicated
                client.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + "cluster1", ReplicaSystemFactory.getPeerConfigurationValue(MockReplicaSystem.class, "100000"));
                break;
            } catch (Exception e) {
                attempts--;
                if (attempts <= 0) {
                    throw e;
                }
                sleepUninterruptibly(2, TimeUnit.SECONDS);
            }
        }
        // Write some data to table1
        writeSomeData(client, table1, 2000, 50);
        // Make sure the replication table is online at this point
        while (!ReplicationTable.isOnline(client)) {
            sleepUninterruptibly(MILLIS_BETWEEN_REPLICATION_TABLE_ONLINE_CHECKS, TimeUnit.MILLISECONDS);
        }
        assertTrue("Replication table was never created", ReplicationTable.isOnline(client));
        // combiner (made by the manager)
        for (int i = 0; i < 10 && !client.tableOperations().listIterators(ReplicationTable.NAME).containsKey(ReplicationTable.COMBINER_NAME); i++) {
            sleepUninterruptibly(2, TimeUnit.SECONDS);
        }
        assertTrue("Combiner was never set on replication table", client.tableOperations().listIterators(ReplicationTable.NAME).containsKey(ReplicationTable.COMBINER_NAME));
        // Trigger the minor compaction, waiting for it to finish.
        // This should write the entry to metadata that the file has data
        client.tableOperations().flush(table1, null, null, true);
        // Make sure that we have one status element, should be a new file
        try (Scanner s = ReplicationTable.getScanner(client)) {
            StatusSection.limit(s);
            Entry<Key, Value> entry = null;
            Status expectedStatus = StatusUtil.openWithUnknownLength();
            attempts = 10;
            // This record will move from new to new with infinite length because of the minc (flush)
            while (entry == null && attempts > 0) {
                try {
                    entry = Iterables.getOnlyElement(s);
                    Status actual = Status.parseFrom(entry.getValue().get());
                    if (actual.getInfiniteEnd() != expectedStatus.getInfiniteEnd()) {
                        entry = null;
                        // the manager process didn't yet fire and write the new mutation, wait for it to do
                        // so and try to read it again
                        Thread.sleep(1000);
                    }
                } catch (NoSuchElementException e) {
                    entry = null;
                    Thread.sleep(500);
                } catch (IllegalArgumentException e) {
                    // saw this contain 2 elements once
                    try (Scanner s2 = ReplicationTable.getScanner(client)) {
                        StatusSection.limit(s2);
                        for (Entry<Key, Value> content : s2) {
                            log.info("{} => {}", content.getKey().toStringNoTruncate(), content.getValue());
                        }
                        throw e;
                    }
                } finally {
                    attempts--;
                }
            }
            assertNotNull("Could not find expected entry in replication table", entry);
            Status actual = Status.parseFrom(entry.getValue().get());
            assertTrue("Expected to find a replication entry that is open with infinite length: " + ProtobufUtil.toString(actual), !actual.getClosed() && actual.getInfiniteEnd());
            // Try a couple of times to watch for the work record to be created
            boolean notFound = true;
            for (int i = 0; i < 10 && notFound; i++) {
                try (Scanner s2 = ReplicationTable.getScanner(client)) {
                    WorkSection.limit(s2);
                    int elementsFound = Iterables.size(s2);
                    if (elementsFound > 0) {
                        assertEquals(1, elementsFound);
                        notFound = false;
                    }
                    Thread.sleep(500);
                }
            }
            // If we didn't find the work record, print the contents of the table
            if (notFound) {
                try (Scanner s2 = ReplicationTable.getScanner(client)) {
                    for (Entry<Key, Value> content : s2) {
                        log.info("{} => {}", content.getKey().toStringNoTruncate(), content.getValue());
                    }
                    assertFalse("Did not find the work entry for the status entry", notFound);
                }
            }
            // Write some more data so that we over-run the single WAL
            writeSomeData(client, table1, 3000, 50);
            log.info("Issued compaction for table");
            client.tableOperations().compact(table1, null, null, true, true);
            log.info("Compaction completed");
            // Manager is creating entries in the replication table from the metadata table every
            // second.
            // Compaction should trigger the record to be written to metadata. Wait a bit to ensure
            // that the manager has time to work.
            Thread.sleep(5000);
            try (Scanner s2 = ReplicationTable.getScanner(client)) {
                StatusSection.limit(s2);
                int numRecords = 0;
                for (Entry<Key, Value> e : s2) {
                    numRecords++;
                    log.info("Found status record {}\t{}", e.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(e.getValue().get())));
                }
                assertEquals(2, numRecords);
            }
            // We should eventually get 2 work records recorded, need to account for a potential delay
            // though
            // might see: status1 -> work1 -> status2 -> (our scans) -> work2
            notFound = true;
            for (int i = 0; i < 10 && notFound; i++) {
                try (Scanner s2 = ReplicationTable.getScanner(client)) {
                    WorkSection.limit(s2);
                    int elementsFound = Iterables.size(s2);
                    if (elementsFound == 2) {
                        notFound = false;
                    }
                    Thread.sleep(500);
                }
            }
            // If we didn't find the work record, print the contents of the table
            if (notFound) {
                try (Scanner s2 = ReplicationTable.getScanner(client)) {
                    for (Entry<Key, Value> content : s2) {
                        log.info("{} => {}", content.getKey().toStringNoTruncate(), content.getValue());
                    }
                    assertFalse("Did not find the work entries for the status entries", notFound);
                }
            }
        }
    }
}

Example 58 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class ReplicationIT method filesClosedAfterUnused.

@Test
public void filesClosedAfterUnused() throws Exception {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        String table = "table";
        Map<String, String> replicate_props = new HashMap<>();
        replicate_props.put(Property.TABLE_REPLICATION.getKey(), "true");
        replicate_props.put(Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
        client.tableOperations().create(table, new NewTableConfiguration().setProperties(replicate_props));
        TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(table));
        assertNotNull(tableId);
        // just sleep
        client.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + "cluster1", ReplicaSystemFactory.getPeerConfigurationValue(MockReplicaSystem.class, "50000"));
        // Write a mutation to make a log file
        try (BatchWriter bw = client.createBatchWriter(table)) {
            Mutation m = new Mutation("one");
            m.put("", "", "");
            bw.addMutation(m);
        }
        // Write another to make sure the logger rolls itself?
        try (BatchWriter bw = client.createBatchWriter(table)) {
            Mutation m = new Mutation("three");
            m.put("", "", "");
            bw.addMutation(m);
        }
        try (Scanner s = client.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            s.fetchColumnFamily(LogColumnFamily.NAME);
            s.setRange(TabletsSection.getRange(tableId));
            Set<String> wals = new HashSet<>();
            for (Entry<Key, Value> entry : s) {
                LogEntry logEntry = LogEntry.fromMetaWalEntry(entry);
                wals.add(new Path(logEntry.filename).toString());
            }
            log.warn("Found wals {}", wals);
            try (BatchWriter bw = client.createBatchWriter(table)) {
                Mutation m = new Mutation("three");
                byte[] bytes = new byte[1024 * 1024];
                m.put("1".getBytes(), new byte[0], bytes);
                m.put("2".getBytes(), new byte[0], bytes);
                m.put("3".getBytes(), new byte[0], bytes);
                m.put("4".getBytes(), new byte[0], bytes);
                m.put("5".getBytes(), new byte[0], bytes);
                bw.addMutation(m);
            }
            client.tableOperations().flush(table, null, null, true);
            while (!ReplicationTable.isOnline(client)) {
                sleepUninterruptibly(MILLIS_BETWEEN_REPLICATION_TABLE_ONLINE_CHECKS, TimeUnit.MILLISECONDS);
            }
            for (int i = 0; i < 10; i++) {
                try (Scanner s2 = client.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
                    s2.fetchColumnFamily(LogColumnFamily.NAME);
                    s2.setRange(TabletsSection.getRange(tableId));
                    for (Entry<Key, Value> entry : s2) {
                        log.info("{}={}", entry.getKey().toStringNoTruncate(), entry.getValue());
                    }
                }
                try (Scanner s3 = ReplicationTable.getScanner(client)) {
                    StatusSection.limit(s3);
                    Text buff = new Text();
                    boolean allReferencedLogsClosed = true;
                    int recordsFound = 0;
                    for (Entry<Key, Value> e : s3) {
                        recordsFound++;
                        allReferencedLogsClosed = true;
                        StatusSection.getFile(e.getKey(), buff);
                        String file = buff.toString();
                        if (wals.contains(file)) {
                            Status stat = Status.parseFrom(e.getValue().get());
                            if (!stat.getClosed()) {
                                log.info("{} wasn't closed", file);
                                allReferencedLogsClosed = false;
                            }
                        }
                    }
                    if (recordsFound > 0 && allReferencedLogsClosed) {
                        return;
                    }
                    Thread.sleep(2000);
                } catch (RuntimeException e) {
                    Throwable cause = e.getCause();
                    if (cause instanceof AccumuloSecurityException) {
                        AccumuloSecurityException ase = (AccumuloSecurityException) cause;
                        switch(ase.getSecurityErrorCode()) {
                            case PERMISSION_DENIED:
                                // We tried to read the replication table before the GRANT went through
                                Thread.sleep(2000);
                                break;
                            default:
                                throw e;
                        }
                    }
                }
            }
            fail("We had a file that was referenced but didn't get closed");
        }
    }
}

Example 59 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class MultiInstanceReplicationIT method dataWasReplicatedToThePeer.

@Test
public void dataWasReplicatedToThePeer() throws Exception {
    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
    peerCfg.setNumTservers(1);
    peerCfg.setInstanceName("peer");
    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
    MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
    peerCluster.start();
    try (AccumuloClient clientManager = Accumulo.newClient().from(getClientProperties()).build();
        AccumuloClient clientPeer = peerCluster.createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD))) {
        ReplicationTable.setOnline(clientManager);
        String peerUserName = "peer", peerPassword = "foo";
        String peerClusterName = "peer";
        clientPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
        // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
        final String managerTable = "manager", peerTable = "peer";
        clientPeer.tableOperations().create(peerTable, new NewTableConfiguration());
        String peerTableId = clientPeer.tableOperations().tableIdMap().get(peerTable);
        assertNotNull(peerTableId);
        clientPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
        // Replicate this table to the peerClusterName in a table with the peerTableId table id
        Map<String, String> props = new HashMap<>();
        props.put(Property.TABLE_REPLICATION.getKey(), "true");
        props.put(Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
        clientManager.tableOperations().create(managerTable, new NewTableConfiguration().setProperties(props));
        String managerTableId = clientManager.tableOperations().tableIdMap().get(managerTable);
        assertNotNull(managerTableId);
        // Write some data to table1
        try (BatchWriter bw = clientManager.createBatchWriter(managerTable)) {
            for (int rows = 0; rows < 5000; rows++) {
                Mutation m = new Mutation(Integer.toString(rows));
                for (int cols = 0; cols < 100; cols++) {
                    String value = Integer.toString(cols);
                    m.put(value, "", value);
                }
                bw.addMutation(m);
            }
        }
        log.info("Wrote all data to manager cluster");
        final Set<String> filesNeedingReplication = clientManager.replicationOperations().referencedFiles(managerTable);
        log.info("Files to replicate: " + filesNeedingReplication);
        for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        cluster.exec(TabletServer.class);
        log.info("TabletServer restarted");
        Iterators.size(ReplicationTable.getScanner(clientManager).iterator());
        log.info("TabletServer is online");
        while (!ReplicationTable.isOnline(clientManager)) {
            log.info("Replication table still offline, waiting");
            Thread.sleep(5000);
        }
        log.info("");
        log.info("Fetching metadata records:");
        try (var scanner = clientManager.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            for (Entry<Key, Value> kv : scanner) {
                if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
                    log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
                } else {
                    log.info("{} {}", kv.getKey().toStringNoTruncate(), kv.getValue());
                }
            }
        }
        log.info("");
        log.info("Fetching replication records:");
        try (var scanner = ReplicationTable.getScanner(clientManager)) {
            for (Entry<Key, Value> kv : scanner) {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
            }
        }
        Future<Boolean> future = executor.submit(() -> {
            long then = System.currentTimeMillis();
            clientManager.replicationOperations().drain(managerTable, filesNeedingReplication);
            long now = System.currentTimeMillis();
            log.info("Drain completed in " + (now - then) + "ms");
            return true;
        });
        try {
            future.get(60, TimeUnit.SECONDS);
        } catch (TimeoutException e) {
            future.cancel(true);
            fail("Drain did not finish within 60 seconds");
        } finally {
            executor.shutdownNow();
        }
        log.info("drain completed");
        log.info("");
        log.info("Fetching metadata records:");
        try (var scanner = clientManager.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            for (Entry<Key, Value> kv : scanner) {
                if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
                    log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
                } else {
                    log.info("{} {}", kv.getKey().toStringNoTruncate(), kv.getValue());
                }
            }
        }
        log.info("");
        log.info("Fetching replication records:");
        try (var scanner = ReplicationTable.getScanner(clientManager)) {
            for (Entry<Key, Value> kv : scanner) {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
            }
        }
        try (Scanner manager = clientManager.createScanner(managerTable, Authorizations.EMPTY);
            Scanner peer = clientPeer.createScanner(peerTable, Authorizations.EMPTY)) {
            Iterator<Entry<Key, Value>> managerIter = manager.iterator(), peerIter = peer.iterator();
            Entry<Key, Value> managerEntry = null, peerEntry = null;
            while (managerIter.hasNext() && peerIter.hasNext()) {
                managerEntry = managerIter.next();
                peerEntry = peerIter.next();
                assertEquals(managerEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0, managerEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
                assertEquals(managerEntry.getValue(), peerEntry.getValue());
            }
            log.info("Last manager entry: {}", managerEntry);
            log.info("Last peer entry: {}", peerEntry);
            assertFalse("Had more data to read from the manager", managerIter.hasNext());
            assertFalse("Had more data to read from the peer", peerIter.hasNext());
        }
    } finally {
        peerCluster.stop();
    }
}

Example 60 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class MultiInstanceReplicationIT method dataReplicatedToCorrectTableWithoutDrain.

@Test
public void dataReplicatedToCorrectTableWithoutDrain() throws Exception {
    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
    peerCfg.setNumTservers(1);
    peerCfg.setInstanceName("peer");
    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
    MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
    peer1Cluster.start();
    try (AccumuloClient clientManager = Accumulo.newClient().from(getClientProperties()).build();
        AccumuloClient clientPeer = peer1Cluster.createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD))) {
        String peerClusterName = "peer";
        String peerUserName = "repl";
        String peerPassword = "passwd";
        // Create a user on the peer for replication to use
        clientPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
        // Configure the credentials we should use to authenticate ourselves to the peer for
        // replication
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
        // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
        String managerTable1 = "manager1", peerTable1 = "peer1", managerTable2 = "manager2", peerTable2 = "peer2";
        // Create tables
        clientPeer.tableOperations().create(peerTable1, new NewTableConfiguration());
        String peerTableId1 = clientPeer.tableOperations().tableIdMap().get(peerTable1);
        assertNotNull(peerTableId1);
        clientPeer.tableOperations().create(peerTable2, new NewTableConfiguration());
        String peerTableId2 = clientPeer.tableOperations().tableIdMap().get(peerTable2);
        assertNotNull(peerTableId2);
        Map<String, String> props1 = new HashMap<>();
        props1.put(Property.TABLE_REPLICATION.getKey(), "true");
        props1.put(Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
        clientManager.tableOperations().create(managerTable1, new NewTableConfiguration().setProperties(props1));
        String managerTableId1 = clientManager.tableOperations().tableIdMap().get(managerTable1);
        assertNotNull(managerTableId1);
        Map<String, String> props2 = new HashMap<>();
        props2.put(Property.TABLE_REPLICATION.getKey(), "true");
        props2.put(Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
        clientManager.tableOperations().create(managerTable2, new NewTableConfiguration().setProperties(props2));
        String managerTableId2 = clientManager.tableOperations().tableIdMap().get(managerTable2);
        assertNotNull(managerTableId2);
        // Give our replication user the ability to write to the tables
        clientPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
        clientPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
        // Replicate this table to the peerClusterName in a table with the peerTableId table id
        clientManager.tableOperations().setProperty(managerTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
        clientManager.tableOperations().setProperty(managerTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
        // Write some data to table1
        try (BatchWriter bw = clientManager.createBatchWriter(managerTable1)) {
            for (int rows = 0; rows < 2500; rows++) {
                Mutation m = new Mutation(managerTable1 + rows);
                for (int cols = 0; cols < 100; cols++) {
                    String value = Integer.toString(cols);
                    m.put(value, "", value);
                }
                bw.addMutation(m);
            }
        }
        // Write some data to table2
        try (BatchWriter bw = clientManager.createBatchWriter(managerTable2)) {
            for (int rows = 0; rows < 2500; rows++) {
                Mutation m = new Mutation(managerTable2 + rows);
                for (int cols = 0; cols < 100; cols++) {
                    String value = Integer.toString(cols);
                    m.put(value, "", value);
                }
                bw.addMutation(m);
            }
        }
        log.info("Wrote all data to manager cluster");
        for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        cluster.exec(TabletServer.class);
        while (!ReplicationTable.isOnline(clientManager)) {
            log.info("Replication table still offline, waiting");
            Thread.sleep(5000);
        }
        // Wait until we fully replicated something
        boolean fullyReplicated = false;
        for (int i = 0; i < 10 && !fullyReplicated; i++) {
            sleepUninterruptibly(2, TimeUnit.SECONDS);
            try (Scanner s = ReplicationTable.getScanner(clientManager)) {
                WorkSection.limit(s);
                for (Entry<Key, Value> entry : s) {
                    Status status = Status.parseFrom(entry.getValue().get());
                    if (StatusUtil.isFullyReplicated(status)) {
                        fullyReplicated |= true;
                    }
                }
            }
        }
        assertNotEquals(0, fullyReplicated);
        // We have to wait for the manager to assign the replication work, a local tserver to process
        // it, and then the remote tserver to replay it
        // Be cautious in how quickly we assert that the data is present on the peer
        long countTable = 0L;
        for (int i = 0; i < 10; i++) {
            for (Entry<Key, Value> entry : clientPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
                countTable++;
                assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(managerTable1));
            }
            log.info("Found {} records in {}", countTable, peerTable1);
            if (countTable == 0L) {
                Thread.sleep(5000);
            } else {
                break;
            }
        }
        assertTrue("Found no records in " + peerTable1 + " in the peer cluster", countTable > 0);
        // Be cautious in how quickly we assert that the data is present on the peer
        for (int i = 0; i < 10; i++) {
            countTable = 0L;
            for (Entry<Key, Value> entry : clientPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
                countTable++;
                assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(managerTable2));
            }
            log.info("Found {} records in {}", countTable, peerTable2);
            if (countTable == 0L) {
                Thread.sleep(5000);
            } else {
                break;
            }
        }
        assertTrue("Found no records in " + peerTable2 + " in the peer cluster", countTable > 0);
    } finally {
        peer1Cluster.stop();
    }
}