Examples with Connector org.apache.accumulo.core.client.Connector used on opensource projects

Example 51 with Connector

use of org.apache.accumulo.core.client.Connector in project accumulo by apache.

the class BulkSplitOptimizationIT method resetConfig.

@After
public void resetConfig() throws Exception {
    if (null != majcDelay) {
        Connector conn = getConnector();
        conn.instanceOperations().setProperty(Property.TSERV_MAJC_DELAY.getKey(), majcDelay);
        getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
        getClusterControl().startAllServers(ServerType.TABLET_SERVER);
    }
}

Example 52 with Connector

use of org.apache.accumulo.core.client.Connector in project accumulo by apache.

the class MultiInstanceReplicationIT method dataWasReplicatedToThePeer.

@Test(timeout = 10 * 60 * 1000)
public void dataWasReplicatedToThePeer() throws Exception {
    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
    peerCfg.setNumTservers(1);
    peerCfg.setInstanceName("peer");
    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
    MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
    peerCluster.start();
    try {
        final Connector connMaster = getConnector();
        final Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
        ReplicationTable.setOnline(connMaster);
        String peerUserName = "peer", peerPassword = "foo";
        String peerClusterName = "peer";
        connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
        // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
        final String masterTable = "master", peerTable = "peer";
        connMaster.tableOperations().create(masterTable);
        String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
        Assert.assertNotNull(masterTableId);
        connPeer.tableOperations().create(peerTable);
        String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
        Assert.assertNotNull(peerTableId);
        connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
        // Replicate this table to the peerClusterName in a table with the peerTableId table id
        connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
        connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
        // Write some data to table1
        BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
        for (int rows = 0; rows < 5000; rows++) {
            Mutation m = new Mutation(Integer.toString(rows));
            for (int cols = 0; cols < 100; cols++) {
                String value = Integer.toString(cols);
                m.put(value, "", value);
            }
            bw.addMutation(m);
        }
        bw.close();
        log.info("Wrote all data to master cluster");
        final Set<String> filesNeedingReplication = connMaster.replicationOperations().referencedFiles(masterTable);
        log.info("Files to replicate: " + filesNeedingReplication);
        for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        cluster.exec(TabletServer.class);
        log.info("TabletServer restarted");
        Iterators.size(ReplicationTable.getScanner(connMaster).iterator());
        log.info("TabletServer is online");
        while (!ReplicationTable.isOnline(connMaster)) {
            log.info("Replication table still offline, waiting");
            Thread.sleep(5000);
        }
        log.info("");
        log.info("Fetching metadata records:");
        for (Entry<Key, Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
            } else {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), kv.getValue());
            }
        }
        log.info("");
        log.info("Fetching replication records:");
        for (Entry<Key, Value> kv : ReplicationTable.getScanner(connMaster)) {
            log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
        }
        Future<Boolean> future = executor.submit(new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                long then = System.currentTimeMillis();
                connMaster.replicationOperations().drain(masterTable, filesNeedingReplication);
                long now = System.currentTimeMillis();
                log.info("Drain completed in " + (now - then) + "ms");
                return true;
            }
        });
        try {
            future.get(60, TimeUnit.SECONDS);
        } catch (TimeoutException e) {
            future.cancel(true);
            Assert.fail("Drain did not finish within 60 seconds");
        } finally {
            executor.shutdownNow();
        }
        log.info("drain completed");
        log.info("");
        log.info("Fetching metadata records:");
        for (Entry<Key, Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
            } else {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), kv.getValue());
            }
        }
        log.info("");
        log.info("Fetching replication records:");
        for (Entry<Key, Value> kv : ReplicationTable.getScanner(connMaster)) {
            log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
        }
        try (Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY);
            Scanner peer = connPeer.createScanner(peerTable, Authorizations.EMPTY)) {
            Iterator<Entry<Key, Value>> masterIter = master.iterator(), peerIter = peer.iterator();
            Entry<Key, Value> masterEntry = null, peerEntry = null;
            while (masterIter.hasNext() && peerIter.hasNext()) {
                masterEntry = masterIter.next();
                peerEntry = peerIter.next();
                Assert.assertEquals(masterEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0, masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
                Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
            }
            log.info("Last master entry: {}", masterEntry);
            log.info("Last peer entry: {}", peerEntry);
            Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
            Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
        }
    } finally {
        peerCluster.stop();
    }
}

Example 53 with Connector

use of org.apache.accumulo.core.client.Connector in project accumulo by apache.

the class MultiInstanceReplicationIT method dataReplicatedToCorrectTableWithoutDrain.

@Test
public void dataReplicatedToCorrectTableWithoutDrain() throws Exception {
    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
    peerCfg.setNumTservers(1);
    peerCfg.setInstanceName("peer");
    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
    MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
    peer1Cluster.start();
    try {
        Connector connMaster = getConnector();
        Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
        String peerClusterName = "peer";
        String peerUserName = "repl";
        String peerPassword = "passwd";
        // Create a user on the peer for replication to use
        connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
        // Configure the credentials we should use to authenticate ourselves to the peer for replication
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
        // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
        String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
        connMaster.tableOperations().create(masterTable1);
        String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
        Assert.assertNotNull(masterTableId1);
        connMaster.tableOperations().create(masterTable2);
        String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
        Assert.assertNotNull(masterTableId2);
        connPeer.tableOperations().create(peerTable1);
        String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
        Assert.assertNotNull(peerTableId1);
        connPeer.tableOperations().create(peerTable2);
        String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
        Assert.assertNotNull(peerTableId2);
        // Give our replication user the ability to write to the tables
        connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
        connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
        // Replicate this table to the peerClusterName in a table with the peerTableId table id
        connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
        connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
        connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
        connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
        // Write some data to table1
        BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
        for (int rows = 0; rows < 2500; rows++) {
            Mutation m = new Mutation(masterTable1 + rows);
            for (int cols = 0; cols < 100; cols++) {
                String value = Integer.toString(cols);
                m.put(value, "", value);
            }
            bw.addMutation(m);
        }
        bw.close();
        // Write some data to table2
        bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
        for (int rows = 0; rows < 2500; rows++) {
            Mutation m = new Mutation(masterTable2 + rows);
            for (int cols = 0; cols < 100; cols++) {
                String value = Integer.toString(cols);
                m.put(value, "", value);
            }
            bw.addMutation(m);
        }
        bw.close();
        log.info("Wrote all data to master cluster");
        for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        cluster.exec(TabletServer.class);
        while (!ReplicationTable.isOnline(connMaster)) {
            log.info("Replication table still offline, waiting");
            Thread.sleep(5000);
        }
        // Wait until we fully replicated something
        boolean fullyReplicated = false;
        for (int i = 0; i < 10 && !fullyReplicated; i++) {
            sleepUninterruptibly(2, TimeUnit.SECONDS);
            try (Scanner s = ReplicationTable.getScanner(connMaster)) {
                WorkSection.limit(s);
                for (Entry<Key, Value> entry : s) {
                    Status status = Status.parseFrom(entry.getValue().get());
                    if (StatusUtil.isFullyReplicated(status)) {
                        fullyReplicated |= true;
                    }
                }
            }
        }
        Assert.assertNotEquals(0, fullyReplicated);
        // We have to wait for the master to assign the replication work, a local tserver to process it, and then the remote tserver to replay it
        // Be cautious in how quickly we assert that the data is present on the peer
        long countTable = 0l;
        for (int i = 0; i < 10; i++) {
            for (Entry<Key, Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
                countTable++;
                Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(masterTable1));
            }
            log.info("Found {} records in {}", countTable, peerTable1);
            if (0l == countTable) {
                Thread.sleep(5000);
            } else {
                break;
            }
        }
        Assert.assertTrue("Found no records in " + peerTable1 + " in the peer cluster", countTable > 0);
        // Be cautious in how quickly we assert that the data is present on the peer
        for (int i = 0; i < 10; i++) {
            countTable = 0l;
            for (Entry<Key, Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
                countTable++;
                Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(masterTable2));
            }
            log.info("Found {} records in {}", countTable, peerTable2);
            if (0l == countTable) {
                Thread.sleep(5000);
            } else {
                break;
            }
        }
        Assert.assertTrue("Found no records in " + peerTable2 + " in the peer cluster", countTable > 0);
    } finally {
        peer1Cluster.stop();
    }
}

Example 54 with Connector

use of org.apache.accumulo.core.client.Connector in project accumulo by apache.

the class MultiTserverReplicationIT method tserverReplicationServicePortsAreAdvertised.

@Test
public void tserverReplicationServicePortsAreAdvertised() throws Exception {
    // Wait for the cluster to be up
    Connector conn = getConnector();
    Instance inst = conn.getInstance();
    // Wait for a tserver to come up to fulfill this request
    conn.tableOperations().create("foo");
    try (Scanner s = conn.createScanner("foo", Authorizations.EMPTY)) {
        Assert.assertEquals(0, Iterables.size(s));
        ZooReader zreader = new ZooReader(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut());
        Set<String> tserverHost = new HashSet<>();
        tserverHost.addAll(zreader.getChildren(ZooUtil.getRoot(inst) + Constants.ZTSERVERS));
        Set<HostAndPort> replicationServices = new HashSet<>();
        for (String tserver : tserverHost) {
            try {
                byte[] portData = zreader.getData(ZooUtil.getRoot(inst) + ReplicationConstants.ZOO_TSERVERS + "/" + tserver, null);
                HostAndPort replAddress = HostAndPort.fromString(new String(portData, UTF_8));
                replicationServices.add(replAddress);
            } catch (Exception e) {
                log.error("Could not find port for {}", tserver, e);
                Assert.fail("Did not find replication port advertisement for " + tserver);
            }
        }
        // Each tserver should also have equial replicaiton services running internally
        Assert.assertEquals("Expected an equal number of replication servicers and tservers", tserverHost.size(), replicationServices.size());
    }
}

Example 55 with Connector

use of org.apache.accumulo.core.client.Connector in project accumulo by apache.

the class ReplicationIT method twoEntriesForTwoTables.

@Test
public void twoEntriesForTwoTables() throws Exception {
    Connector conn = getConnector();
    String table1 = "table1", table2 = "table2";
    // replication shouldn't exist when we begin
    Assert.assertFalse("Replication table already online at the beginning of the test", ReplicationTable.isOnline(conn));
    // Create two tables
    conn.tableOperations().create(table1);
    conn.tableOperations().create(table2);
    conn.securityOperations().grantTablePermission("root", ReplicationTable.NAME, TablePermission.READ);
    // wait for permission to propagate
    Thread.sleep(5000);
    // Enable replication on table1
    conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION.getKey(), "true");
    // Despite having replication on, we shouldn't have any need to write a record to it (and bring it online)
    Assert.assertFalse(ReplicationTable.isOnline(conn));
    // Write some data to table1
    writeSomeData(conn, table1, 50, 50);
    // After writing data, we'll get a replication table online
    while (!ReplicationTable.isOnline(conn)) {
        sleepUninterruptibly(MILLIS_BETWEEN_REPLICATION_TABLE_ONLINE_CHECKS, TimeUnit.MILLISECONDS);
    }
    Assert.assertTrue(ReplicationTable.isOnline(conn));
    // Verify that we found a single replication record that's for table1
    Entry<Key, Value> entry;
    try (Scanner s = ReplicationTable.getScanner(conn)) {
        StatusSection.limit(s);
        for (int i = 0; i < 5; i++) {
            if (Iterators.size(s.iterator()) == 1) {
                break;
            }
            Thread.sleep(1000);
        }
        entry = Iterators.getOnlyElement(s.iterator());
    }
    // We should at least find one status record for this table, we might find a second if another log was started from ingesting the data
    Assert.assertEquals("Expected to find replication entry for " + table1, conn.tableOperations().tableIdMap().get(table1), entry.getKey().getColumnQualifier().toString());
    // Enable replication on table2
    conn.tableOperations().setProperty(table2, Property.TABLE_REPLICATION.getKey(), "true");
    // Write some data to table2
    writeSomeData(conn, table2, 50, 50);
    // After the commit on these mutations, we'll get a replication entry in accumulo.metadata for table2
    // Don't want to compact table2 as it ultimately cause the entry in accumulo.metadata to be removed before we can verify it's there
    Set<String> tableIds = Sets.newHashSet(conn.tableOperations().tableIdMap().get(table1), conn.tableOperations().tableIdMap().get(table2));
    Set<String> tableIdsForMetadata = Sets.newHashSet(tableIds);
    List<Entry<Key, Value>> records = new ArrayList<>();
    try (Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        s.setRange(MetadataSchema.ReplicationSection.getRange());
        for (Entry<Key, Value> metadata : s) {
            records.add(metadata);
            log.debug("Meta: {} => {}", metadata.getKey().toStringNoTruncate(), metadata.getValue().toString());
        }
        Assert.assertEquals("Expected to find 2 records, but actually found " + records, 2, records.size());
        for (Entry<Key, Value> metadata : records) {
            Assert.assertTrue("Expected record to be in metadata but wasn't " + metadata.getKey().toStringNoTruncate() + ", tableIds remaining " + tableIdsForMetadata, tableIdsForMetadata.remove(metadata.getKey().getColumnQualifier().toString()));
        }
        Assert.assertTrue("Expected that we had removed all metadata entries " + tableIdsForMetadata, tableIdsForMetadata.isEmpty());
        // Should be creating these records in replication table from metadata table every second
        Thread.sleep(5000);
    }
    // Verify that we found two replication records: one for table1 and one for table2
    try (Scanner s = ReplicationTable.getScanner(conn)) {
        StatusSection.limit(s);
        Iterator<Entry<Key, Value>> iter = s.iterator();
        Assert.assertTrue("Found no records in replication table", iter.hasNext());
        entry = iter.next();
        Assert.assertTrue("Expected to find element in replication table", tableIds.remove(entry.getKey().getColumnQualifier().toString()));
        Assert.assertTrue("Expected to find two elements in replication table, only found one ", iter.hasNext());
        entry = iter.next();
        Assert.assertTrue("Expected to find element in replication table", tableIds.remove(entry.getKey().getColumnQualifier().toString()));
        Assert.assertFalse("Expected to only find two elements in replication table", iter.hasNext());
    }
}