Examples with AccumuloClient org.apache.accumulo.core.client.AccumuloClient used on opensource projects

Example 51 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class UnorderedWorkAssignerReplicationIT method dataReplicatedToCorrectTableWithoutDrain.

@Test
public void dataReplicatedToCorrectTableWithoutDrain() throws Exception {
    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
    peerCfg.setNumTservers(1);
    peerCfg.setInstanceName("peer");
    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
    MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
    peer1Cluster.start();
    try (AccumuloClient clientManager = Accumulo.newClient().from(getClientProperties()).build();
        AccumuloClient clientPeer = peer1Cluster.createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD))) {
        String peerClusterName = "peer";
        String peerUserName = "repl";
        String peerPassword = "passwd";
        // Create a user on the peer for replication to use
        clientPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
        // Configure the credentials we should use to authenticate ourselves to the peer for
        // replication
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
        // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
        String managerTable1 = "manager1", peerTable1 = "peer1", managerTable2 = "manager2", peerTable2 = "peer2";
        clientPeer.tableOperations().create(peerTable1, new NewTableConfiguration());
        String peerTableId1 = clientPeer.tableOperations().tableIdMap().get(peerTable1);
        assertNotNull(peerTableId1);
        clientPeer.tableOperations().create(peerTable2, new NewTableConfiguration());
        String peerTableId2 = clientPeer.tableOperations().tableIdMap().get(peerTable2);
        assertNotNull(peerTableId2);
        Map<String, String> props1 = new HashMap<>();
        props1.put(Property.TABLE_REPLICATION.getKey(), "true");
        props1.put(Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
        clientManager.tableOperations().create(managerTable1, new NewTableConfiguration().setProperties(props1));
        String managerTableId1 = clientManager.tableOperations().tableIdMap().get(managerTable1);
        assertNotNull(managerTableId1);
        Map<String, String> props2 = new HashMap<>();
        props2.put(Property.TABLE_REPLICATION.getKey(), "true");
        props2.put(Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
        clientManager.tableOperations().create(managerTable2, new NewTableConfiguration().setProperties(props2));
        String managerTableId2 = clientManager.tableOperations().tableIdMap().get(managerTable2);
        assertNotNull(managerTableId2);
        // Give our replication user the ability to write to the tables
        clientPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
        clientPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
        // Wait for zookeeper updates (configuration) to propagate
        sleepUninterruptibly(3, TimeUnit.SECONDS);
        // Write some data to table1
        try (BatchWriter bw = clientManager.createBatchWriter(managerTable1)) {
            for (int rows = 0; rows < 2500; rows++) {
                Mutation m = new Mutation(managerTable1 + rows);
                for (int cols = 0; cols < 100; cols++) {
                    String value = Integer.toString(cols);
                    m.put(value, "", value);
                }
                bw.addMutation(m);
            }
        }
        // Write some data to table2
        try (BatchWriter bw = clientManager.createBatchWriter(managerTable2)) {
            for (int rows = 0; rows < 2500; rows++) {
                Mutation m = new Mutation(managerTable2 + rows);
                for (int cols = 0; cols < 100; cols++) {
                    String value = Integer.toString(cols);
                    m.put(value, "", value);
                }
                bw.addMutation(m);
            }
        }
        log.info("Wrote all data to manager cluster");
        while (!ReplicationTable.isOnline(clientManager)) {
            Thread.sleep(500);
        }
        for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        cluster.exec(TabletServer.class);
        // Wait until we fully replicated something
        boolean fullyReplicated = false;
        for (int i = 0; i < 10 && !fullyReplicated; i++) {
            sleepUninterruptibly(timeoutFactor * 2, TimeUnit.SECONDS);
            try (Scanner s = ReplicationTable.getScanner(clientManager)) {
                WorkSection.limit(s);
                for (Entry<Key, Value> entry : s) {
                    Status status = Status.parseFrom(entry.getValue().get());
                    if (StatusUtil.isFullyReplicated(status)) {
                        fullyReplicated |= true;
                    }
                }
            }
        }
        assertNotEquals(0, fullyReplicated);
        long countTable = 0L;
        // Check a few times
        for (int i = 0; i < 10; i++) {
            countTable = 0L;
            for (Entry<Key, Value> entry : clientPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
                countTable++;
                assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(managerTable1));
            }
            log.info("Found {} records in {}", countTable, peerTable1);
            if (countTable > 0) {
                break;
            }
            Thread.sleep(2000);
        }
        assertTrue("Did not find any records in " + peerTable1 + " on peer", countTable > 0);
        for (int i = 0; i < 10; i++) {
            countTable = 0L;
            for (Entry<Key, Value> entry : clientPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
                countTable++;
                assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(managerTable2));
            }
            log.info("Found {} records in {}", countTable, peerTable2);
            if (countTable > 0) {
                break;
            }
            Thread.sleep(2000);
        }
        assertTrue("Did not find any records in " + peerTable2 + " on peer", countTable > 0);
    } finally {
        peer1Cluster.stop();
    }
}

Example 52 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class UnorderedWorkAssignerReplicationIT method dataWasReplicatedToThePeer.

@Test
public void dataWasReplicatedToThePeer() throws Exception {
    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
    peerCfg.setNumTservers(1);
    peerCfg.setInstanceName("peer");
    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
    MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
    peerCluster.start();
    try (AccumuloClient clientManager = Accumulo.newClient().from(getClientProperties()).build();
        AccumuloClient clientPeer = peerCluster.createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD))) {
        ReplicationTable.setOnline(clientManager);
        String peerUserName = "peer", peerPassword = "foo";
        String peerClusterName = "peer";
        clientPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
        // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
        final String managerTable = "manager", peerTable = "peer";
        clientPeer.tableOperations().create(peerTable);
        String peerTableId = clientPeer.tableOperations().tableIdMap().get(peerTable);
        assertNotNull(peerTableId);
        Map<String, String> props = new HashMap<>();
        props.put(Property.TABLE_REPLICATION.getKey(), "true");
        props.put(Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
        clientManager.tableOperations().create(managerTable, new NewTableConfiguration().setProperties(props));
        String managerTableId = clientManager.tableOperations().tableIdMap().get(managerTable);
        assertNotNull(managerTableId);
        clientPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
        // Wait for zookeeper updates (configuration) to propagate
        sleepUninterruptibly(3, TimeUnit.SECONDS);
        // Write some data to table1
        try (BatchWriter bw = clientManager.createBatchWriter(managerTable)) {
            for (int rows = 0; rows < 5000; rows++) {
                Mutation m = new Mutation(Integer.toString(rows));
                for (int cols = 0; cols < 100; cols++) {
                    String value = Integer.toString(cols);
                    m.put(value, "", value);
                }
                bw.addMutation(m);
            }
        }
        log.info("Wrote all data to manager cluster");
        final Set<String> filesNeedingReplication = clientManager.replicationOperations().referencedFiles(managerTable);
        for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        cluster.exec(TabletServer.class);
        log.info("TabletServer restarted");
        Iterators.size(ReplicationTable.getScanner(clientManager).iterator());
        log.info("TabletServer is online");
        log.info("");
        log.info("Fetching metadata records:");
        try (var scanner = clientManager.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            for (Entry<Key, Value> kv : scanner) {
                if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
                    log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
                } else {
                    log.info("{} {}", kv.getKey().toStringNoTruncate(), kv.getValue());
                }
            }
        }
        log.info("");
        log.info("Fetching replication records:");
        try (var scanner = ReplicationTable.getScanner(clientManager)) {
            for (Entry<Key, Value> kv : scanner) {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
            }
        }
        Future<Boolean> future = executor.submit(() -> {
            clientManager.replicationOperations().drain(managerTable, filesNeedingReplication);
            log.info("Drain completed");
            return true;
        });
        long timeoutSeconds = timeoutFactor * 30L;
        try {
            future.get(timeoutSeconds, TimeUnit.SECONDS);
        } catch (TimeoutException e) {
            future.cancel(true);
            fail("Drain did not finish within " + timeoutSeconds + " seconds");
        }
        log.info("drain completed");
        log.info("");
        log.info("Fetching metadata records:");
        try (var scanner = clientManager.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            for (Entry<Key, Value> kv : scanner) {
                if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
                    log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
                } else {
                    log.info("{} {}", kv.getKey().toStringNoTruncate(), kv.getValue());
                }
            }
        }
        log.info("");
        log.info("Fetching replication records:");
        try (var scanner = ReplicationTable.getScanner(clientManager)) {
            for (Entry<Key, Value> kv : scanner) {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
            }
        }
        try (Scanner manager = clientManager.createScanner(managerTable, Authorizations.EMPTY);
            Scanner peer = clientPeer.createScanner(peerTable, Authorizations.EMPTY)) {
            Iterator<Entry<Key, Value>> managerIter = manager.iterator(), peerIter = peer.iterator();
            Entry<Key, Value> managerEntry = null, peerEntry = null;
            while (managerIter.hasNext() && peerIter.hasNext()) {
                managerEntry = managerIter.next();
                peerEntry = peerIter.next();
                assertEquals(managerEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0, managerEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
                assertEquals(managerEntry.getValue(), peerEntry.getValue());
            }
            log.info("Last manager entry: {}", managerEntry);
            log.info("Last peer entry: {}", peerEntry);
            assertFalse("Had more data to read from the manager", managerIter.hasNext());
            assertFalse("Had more data to read from the peer", peerIter.hasNext());
        }
    } finally {
        peerCluster.stop();
    }
}

Example 53 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class ReplicationIT method correctRecordsCompleteFile.

@Test
public void correctRecordsCompleteFile() throws Exception {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        String table = "table1";
        client.tableOperations().create(table, new NewTableConfiguration().setProperties(singletonMap(Property.TABLE_REPLICATION.getKey(), "true")));
        try (BatchWriter bw = client.createBatchWriter(table)) {
            for (int i = 0; i < 10; i++) {
                Mutation m = new Mutation(Integer.toString(i));
                m.put(new byte[0], new byte[0], new byte[0]);
                bw.addMutation(m);
            }
        }
        // After writing data, we'll get a replication table online
        while (!ReplicationTable.isOnline(client)) {
            sleepUninterruptibly(MILLIS_BETWEEN_REPLICATION_TABLE_ONLINE_CHECKS, TimeUnit.MILLISECONDS);
        }
        assertTrue("Replication table did not exist", ReplicationTable.isOnline(client));
        for (int i = 0; i < 5; i++) {
            if (client.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ)) {
                break;
            }
            log.info("Could not read replication table, waiting and will retry");
            Thread.sleep(2000);
        }
        assertTrue("'root' user could not read the replication table", client.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ));
        Set<String> replRows = new HashSet<>();
        int attempts = 5;
        while (replRows.isEmpty() && attempts > 0) {
            try (Scanner scanner = ReplicationTable.getScanner(client)) {
                StatusSection.limit(scanner);
                for (Entry<Key, Value> entry : scanner) {
                    Key k = entry.getKey();
                    String fileUri = k.getRow().toString();
                    try {
                        new URI(fileUri);
                    } catch (URISyntaxException e) {
                        fail("Expected a valid URI: " + fileUri);
                    }
                    replRows.add(fileUri);
                }
            }
        }
        Set<String> wals = new HashSet<>();
        attempts = 5;
        while (wals.isEmpty() && attempts > 0) {
            WalStateManager markers = new WalStateManager(getServerContext());
            for (Entry<Path, WalState> entry : markers.getAllState().entrySet()) {
                wals.add(entry.getKey().toString());
            }
            attempts--;
        }
        // We only have one file that should need replication (no trace table)
        // We should find an entry in tablet and in the repl row
        assertEquals("Rows found: " + replRows, 1, replRows.size());
        // There should only be one extra WALog that replication doesn't know about
        replRows.removeAll(wals);
        assertEquals(2, wals.size());
        assertEquals(0, replRows.size());
    }
}

Example 54 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class ReplicationIT method noRecordsWithoutReplication.

@Test
public void noRecordsWithoutReplication() throws Exception {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        List<String> tables = new ArrayList<>();
        // replication shouldn't be online when we begin
        assertFalse(ReplicationTable.isOnline(client));
        for (int i = 0; i < 5; i++) {
            String name = "table" + i;
            tables.add(name);
            client.tableOperations().create(name);
        }
        // nor after we create some tables (that aren't being replicated)
        assertFalse(ReplicationTable.isOnline(client));
        for (String table : tables) {
            writeSomeData(client, table, 5, 5);
        }
        // After writing data, still no replication table
        assertFalse(ReplicationTable.isOnline(client));
        for (String table : tables) {
            client.tableOperations().compact(table, null, null, true, true);
        }
        // After compacting data, still no replication table
        assertFalse(ReplicationTable.isOnline(client));
        for (String table : tables) {
            client.tableOperations().delete(table);
        }
        // After deleting tables, still no replication table
        assertFalse(ReplicationTable.isOnline(client));
    }
}

Example 55 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class ReplicationIT method replicationEntriesPrecludeWalDeletion.

@Test
public void replicationEntriesPrecludeWalDeletion() throws Exception {
    final ServerContext context = getServerContext();
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        String table1 = "table1", table2 = "table2", table3 = "table3";
        final Multimap<String, TableId> logs = HashMultimap.create();
        final AtomicBoolean keepRunning = new AtomicBoolean(true);
        Thread t = new Thread(() -> {
            // when that happens
            while (keepRunning.get()) {
                try {
                    logs.putAll(getAllLogs(client, context));
                } catch (Exception e) {
                    log.error("Error getting logs", e);
                }
            }
        });
        t.start();
        HashMap<String, String> replicate_props = new HashMap<>();
        replicate_props.put(Property.TABLE_REPLICATION.getKey(), "true");
        replicate_props.put(Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
        client.tableOperations().create(table1, new NewTableConfiguration().setProperties(replicate_props));
        Thread.sleep(2000);
        // Write some data to table1
        writeSomeData(client, table1, 200, 500);
        client.tableOperations().create(table2, new NewTableConfiguration().setProperties(replicate_props));
        Thread.sleep(2000);
        writeSomeData(client, table2, 200, 500);
        client.tableOperations().create(table3, new NewTableConfiguration().setProperties(replicate_props));
        Thread.sleep(2000);
        writeSomeData(client, table3, 200, 500);
        // Force a write to metadata for the data written
        for (String table : Arrays.asList(table1, table2, table3)) {
            client.tableOperations().flush(table, null, null, true);
        }
        keepRunning.set(false);
        t.join(5000);
        // The manager is only running every second to create records in the replication table from
        // the
        // metadata table
        // Sleep a sufficient amount of time to ensure that we get the straggling WALs that might have
        // been created at the end
        Thread.sleep(5000);
        Set<String> replFiles = getReferencesToFilesToBeReplicated(client);
        // We might have a WAL that was use solely for the replication table
        // We want to remove that from our list as it should not appear in the replication table
        String replicationTableId = client.tableOperations().tableIdMap().get(ReplicationTable.NAME);
        Iterator<Entry<String, TableId>> observedLogs = logs.entries().iterator();
        while (observedLogs.hasNext()) {
            Entry<String, TableId> observedLog = observedLogs.next();
            if (replicationTableId.equals(observedLog.getValue().canonical())) {
                log.info("Removing {} because its tableId is for the replication table", observedLog);
                observedLogs.remove();
            }
        }
        // We should have *some* reference to each log that was seen in the metadata table
        // They might not yet all be closed though (might be newfile)
        assertTrue("Metadata log distribution: " + logs + "replFiles " + replFiles, logs.keySet().containsAll(replFiles));
        assertTrue("Difference between replication entries and current logs is bigger than one", logs.keySet().size() - replFiles.size() <= 1);
        final Configuration conf = new Configuration();
        for (String replFile : replFiles) {
            Path p = new Path(replFile);
            FileSystem fs = p.getFileSystem(conf);
            if (!fs.exists(p)) {
                // double-check: the garbage collector can be fast
                Set<String> currentSet = getReferencesToFilesToBeReplicated(client);
                log.info("Current references {}", currentSet);
                log.info("Looking for reference to {}", replFile);
                log.info("Contains? {}", currentSet.contains(replFile));
                assertTrue("File does not exist anymore, it was likely incorrectly garbage collected: " + p, !currentSet.contains(replFile));
            }
        }
    }
}