Examples with MiniAccumuloClusterImpl org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl used on opensource projects

Example 31 with MiniAccumuloClusterImpl

use of org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl in project accumulo by apache.

the class UnorderedWorkAssignerReplicationIT method dataWasReplicatedToThePeerWithoutDrain.

@Test
public void dataWasReplicatedToThePeerWithoutDrain() throws Exception {
    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
    peerCfg.setNumTservers(1);
    peerCfg.setInstanceName("peer");
    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
    MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
    peerCluster.start();
    Connector connMaster = getConnector();
    Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
    String peerUserName = "repl";
    String peerPassword = "passwd";
    // Create a user on the peer for replication to use
    connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
    String peerClusterName = "peer";
    // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
    connMaster.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
    // Configure the credentials we should use to authenticate ourselves to the peer for replication
    connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
    connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
    String masterTable = "master", peerTable = "peer";
    connMaster.tableOperations().create(masterTable);
    String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
    Assert.assertNotNull(masterTableId);
    connPeer.tableOperations().create(peerTable);
    String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
    Assert.assertNotNull(peerTableId);
    // Give our replication user the ability to write to the table
    connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
    // Replicate this table to the peerClusterName in a table with the peerTableId table id
    connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
    connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
    // Write some data to table1
    BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
    for (int rows = 0; rows < 5000; rows++) {
        Mutation m = new Mutation(Integer.toString(rows));
        for (int cols = 0; cols < 100; cols++) {
            String value = Integer.toString(cols);
            m.put(value, "", value);
        }
        bw.addMutation(m);
    }
    bw.close();
    log.info("Wrote all data to master cluster");
    Set<String> files = connMaster.replicationOperations().referencedFiles(masterTable);
    for (String s : files) {
        log.info("Found referenced file for {}: {}", masterTable, s);
    }
    for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
        cluster.killProcess(ServerType.TABLET_SERVER, proc);
    }
    cluster.exec(TabletServer.class);
    Iterators.size(connMaster.createScanner(masterTable, Authorizations.EMPTY).iterator());
    for (Entry<Key, Value> kv : connMaster.createScanner(ReplicationTable.NAME, Authorizations.EMPTY)) {
        log.debug("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
    }
    connMaster.replicationOperations().drain(masterTable, files);
    try (Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY);
        Scanner peer = connPeer.createScanner(peerTable, Authorizations.EMPTY)) {
        Iterator<Entry<Key, Value>> masterIter = master.iterator(), peerIter = peer.iterator();
        Assert.assertTrue("No data in master table", masterIter.hasNext());
        Assert.assertTrue("No data in peer table", peerIter.hasNext());
        while (masterIter.hasNext() && peerIter.hasNext()) {
            Entry<Key, Value> masterEntry = masterIter.next(), peerEntry = peerIter.next();
            Assert.assertEquals(peerEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0, masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
            Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
        }
        Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
        Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
    }
    peerCluster.stop();
}

Example 32 with MiniAccumuloClusterImpl

use of org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl in project accumulo by apache.

the class CyclicReplicationIT method dataIsNotOverReplicated.

@Test
public void dataIsNotOverReplicated() throws Exception {
    File master1Dir = createTestDir("master1"), master2Dir = createTestDir("master2");
    String password = "password";
    MiniAccumuloConfigImpl master1Cfg;
    MiniAccumuloClusterImpl master1Cluster;
    while (true) {
        master1Cfg = new MiniAccumuloConfigImpl(master1Dir, password);
        master1Cfg.setNumTservers(1);
        master1Cfg.setInstanceName("master1");
        // Set up SSL if needed
        ConfigurableMacBase.configureForEnvironment(master1Cfg, this.getClass(), ConfigurableMacBase.getSslDir(master1Dir));
        master1Cfg.setProperty(Property.REPLICATION_NAME, master1Cfg.getInstanceName());
        master1Cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "5M");
        master1Cfg.setProperty(Property.REPLICATION_THREADCHECK, "5m");
        master1Cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "1s");
        master1Cfg.setProperty(Property.MASTER_REPLICATION_SCAN_INTERVAL, "1s");
        master1Cluster = new MiniAccumuloClusterImpl(master1Cfg);
        setCoreSite(master1Cluster);
        try {
            master1Cluster.start();
            break;
        } catch (ZooKeeperBindException e) {
            log.warn("Failed to start ZooKeeper on {}, will retry", master1Cfg.getZooKeeperPort());
        }
    }
    MiniAccumuloConfigImpl master2Cfg;
    MiniAccumuloClusterImpl master2Cluster;
    while (true) {
        master2Cfg = new MiniAccumuloConfigImpl(master2Dir, password);
        master2Cfg.setNumTservers(1);
        master2Cfg.setInstanceName("master2");
        // Set up SSL if needed. Need to share the same SSL truststore as master1
        this.updatePeerConfigFromPrimary(master1Cfg, master2Cfg);
        master2Cfg.setProperty(Property.REPLICATION_NAME, master2Cfg.getInstanceName());
        master2Cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "5M");
        master2Cfg.setProperty(Property.REPLICATION_THREADCHECK, "5m");
        master2Cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "1s");
        master2Cfg.setProperty(Property.MASTER_REPLICATION_SCAN_INTERVAL, "1s");
        master2Cluster = new MiniAccumuloClusterImpl(master2Cfg);
        setCoreSite(master2Cluster);
        try {
            master2Cluster.start();
            break;
        } catch (ZooKeeperBindException e) {
            log.warn("Failed to start ZooKeeper on {}, will retry", master2Cfg.getZooKeeperPort());
        }
    }
    try {
        Connector connMaster1 = master1Cluster.getConnector("root", new PasswordToken(password)), connMaster2 = master2Cluster.getConnector("root", new PasswordToken(password));
        String master1UserName = "master1", master1Password = "foo";
        String master2UserName = "master2", master2Password = "bar";
        String master1Table = master1Cluster.getInstanceName(), master2Table = master2Cluster.getInstanceName();
        connMaster1.securityOperations().createLocalUser(master1UserName, new PasswordToken(master1Password));
        connMaster2.securityOperations().createLocalUser(master2UserName, new PasswordToken(master2Password));
        // Configure the credentials we should use to authenticate ourselves to the peer for replication
        connMaster1.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + master2Cluster.getInstanceName(), master2UserName);
        connMaster1.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + master2Cluster.getInstanceName(), master2Password);
        connMaster2.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + master1Cluster.getInstanceName(), master1UserName);
        connMaster2.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + master1Cluster.getInstanceName(), master1Password);
        connMaster1.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + master2Cluster.getInstanceName(), ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(master2Cluster.getInstanceName(), master2Cluster.getZooKeepers())));
        connMaster2.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + master1Cluster.getInstanceName(), ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(master1Cluster.getInstanceName(), master1Cluster.getZooKeepers())));
        connMaster1.tableOperations().create(master1Table, new NewTableConfiguration().withoutDefaultIterators());
        String master1TableId = connMaster1.tableOperations().tableIdMap().get(master1Table);
        Assert.assertNotNull(master1TableId);
        connMaster2.tableOperations().create(master2Table, new NewTableConfiguration().withoutDefaultIterators());
        String master2TableId = connMaster2.tableOperations().tableIdMap().get(master2Table);
        Assert.assertNotNull(master2TableId);
        // Replicate master1 in the master1 cluster to master2 in the master2 cluster
        connMaster1.tableOperations().setProperty(master1Table, Property.TABLE_REPLICATION.getKey(), "true");
        connMaster1.tableOperations().setProperty(master1Table, Property.TABLE_REPLICATION_TARGET.getKey() + master2Cluster.getInstanceName(), master2TableId);
        // Replicate master2 in the master2 cluster to master1 in the master2 cluster
        connMaster2.tableOperations().setProperty(master2Table, Property.TABLE_REPLICATION.getKey(), "true");
        connMaster2.tableOperations().setProperty(master2Table, Property.TABLE_REPLICATION_TARGET.getKey() + master1Cluster.getInstanceName(), master1TableId);
        // Give our replication user the ability to write to the respective table
        connMaster1.securityOperations().grantTablePermission(master1UserName, master1Table, TablePermission.WRITE);
        connMaster2.securityOperations().grantTablePermission(master2UserName, master2Table, TablePermission.WRITE);
        IteratorSetting summingCombiner = new IteratorSetting(50, SummingCombiner.class);
        SummingCombiner.setEncodingType(summingCombiner, Type.STRING);
        SummingCombiner.setCombineAllColumns(summingCombiner, true);
        // Set a combiner on both instances that will sum multiple values
        // We can use this to verify that the mutation was not sent multiple times
        connMaster1.tableOperations().attachIterator(master1Table, summingCombiner);
        connMaster2.tableOperations().attachIterator(master2Table, summingCombiner);
        // Write a single entry
        BatchWriter bw = connMaster1.createBatchWriter(master1Table, new BatchWriterConfig());
        Mutation m = new Mutation("row");
        m.put("count", "", "1");
        bw.addMutation(m);
        bw.close();
        Set<String> files = connMaster1.replicationOperations().referencedFiles(master1Table);
        log.info("Found {} that need replication from master1", files);
        // Kill and restart the tserver to close the WAL on master1
        for (ProcessReference proc : master1Cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            master1Cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        master1Cluster.exec(TabletServer.class);
        log.info("Restarted tserver on master1");
        // Try to avoid ACCUMULO-2964
        Thread.sleep(1000);
        // Sanity check that the element is there on master1
        Entry<Key, Value> entry;
        try (Scanner s = connMaster1.createScanner(master1Table, Authorizations.EMPTY)) {
            entry = Iterables.getOnlyElement(s);
            Assert.assertEquals("1", entry.getValue().toString());
            // Wait for this table to replicate
            connMaster1.replicationOperations().drain(master1Table, files);
            Thread.sleep(5000);
        }
        // Check that the element made it to master2 only once
        try (Scanner s = connMaster2.createScanner(master2Table, Authorizations.EMPTY)) {
            entry = Iterables.getOnlyElement(s);
            Assert.assertEquals("1", entry.getValue().toString());
            // Wait for master2 to finish replicating it back
            files = connMaster2.replicationOperations().referencedFiles(master2Table);
            // Kill and restart the tserver to close the WAL on master2
            for (ProcessReference proc : master2Cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
                master2Cluster.killProcess(ServerType.TABLET_SERVER, proc);
            }
            master2Cluster.exec(TabletServer.class);
            // Try to avoid ACCUMULO-2964
            Thread.sleep(1000);
        }
        // Check that the element made it to master2 only once
        try (Scanner s = connMaster2.createScanner(master2Table, Authorizations.EMPTY)) {
            entry = Iterables.getOnlyElement(s);
            Assert.assertEquals("1", entry.getValue().toString());
            connMaster2.replicationOperations().drain(master2Table, files);
            Thread.sleep(5000);
        }
        // Verify that the entry wasn't sent back to master1
        try (Scanner s = connMaster1.createScanner(master1Table, Authorizations.EMPTY)) {
            entry = Iterables.getOnlyElement(s);
            Assert.assertEquals("1", entry.getValue().toString());
        }
    } finally {
        master1Cluster.stop();
        master2Cluster.stop();
    }
}

Example 33 with MiniAccumuloClusterImpl

use of org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl in project accumulo by apache.

the class AccumuloClusterHarness method setupCluster.

@Before
public void setupCluster() throws Exception {
    // Before we try to instantiate the cluster, check to see if the test even wants to run against this type of cluster
    Assume.assumeTrue(canRunTest(type));
    switch(type) {
        case MINI:
            MiniClusterHarness miniClusterHarness = new MiniClusterHarness();
            // Intrinsically performs the callback to let tests alter MiniAccumuloConfig and core-site.xml
            MiniAccumuloClusterImpl impl = miniClusterHarness.create(this, getAdminToken(), krb);
            cluster = impl;
            // MAC makes a ClientConf for us, just set it
            ((AccumuloMiniClusterConfiguration) clusterConf).setClientConf(impl.getClientConfig());
            // Login as the "root" user
            if (null != krb) {
                ClusterUser rootUser = krb.getRootUser();
                // Log in the 'client' user
                UserGroupInformation.loginUserFromKeytab(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
            }
            break;
        case STANDALONE:
            StandaloneAccumuloClusterConfiguration conf = (StandaloneAccumuloClusterConfiguration) clusterConf;
            ClientConfiguration clientConf = conf.getClientConf();
            StandaloneAccumuloCluster standaloneCluster = new StandaloneAccumuloCluster(conf.getInstance(), clientConf, conf.getTmpDirectory(), conf.getUsers());
            // If these are provided in the configuration, pass them into the cluster
            standaloneCluster.setAccumuloHome(conf.getAccumuloHome());
            standaloneCluster.setClientAccumuloConfDir(conf.getClientAccumuloConfDir());
            standaloneCluster.setServerAccumuloConfDir(conf.getServerAccumuloConfDir());
            standaloneCluster.setHadoopConfDir(conf.getHadoopConfDir());
            standaloneCluster.setServerCmdPrefix(conf.getServerCmdPrefix());
            standaloneCluster.setClientCmdPrefix(conf.getClientCmdPrefix());
            // For SASL, we need to get the Hadoop configuration files as well otherwise UGI will log in as SIMPLE instead of KERBEROS
            Configuration hadoopConfiguration = standaloneCluster.getHadoopConfiguration();
            if (clientConf.hasSasl()) {
                UserGroupInformation.setConfiguration(hadoopConfiguration);
                // Login as the admin user to start the tests
                UserGroupInformation.loginUserFromKeytab(conf.getAdminPrincipal(), conf.getAdminKeytab().getAbsolutePath());
            }
            // Set the implementation
            cluster = standaloneCluster;
            break;
        default:
            throw new RuntimeException("Unhandled type");
    }
    if (type.isDynamic()) {
        cluster.start();
    } else {
        log.info("Removing tables which appear to be from a previous test run");
        cleanupTables();
        log.info("Removing users which appear to be from a previous test run");
        cleanupUsers();
    }
    switch(type) {
        case MINI:
            if (null != krb) {
                final String traceTable = Property.TRACE_TABLE.getDefaultValue();
                final ClusterUser systemUser = krb.getAccumuloServerUser(), rootUser = krb.getRootUser();
                // Login as the trace user
                UserGroupInformation.loginUserFromKeytab(systemUser.getPrincipal(), systemUser.getKeytab().getAbsolutePath());
                // Open a connector as the system user (ensures the user will exist for us to assign permissions to)
                UserGroupInformation.loginUserFromKeytab(systemUser.getPrincipal(), systemUser.getKeytab().getAbsolutePath());
                Connector conn = cluster.getConnector(systemUser.getPrincipal(), new KerberosToken());
                // Then, log back in as the "root" user and do the grant
                UserGroupInformation.loginUserFromKeytab(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
                conn = getConnector();
                // Create the trace table
                conn.tableOperations().create(traceTable);
                // Trace user (which is the same kerberos principal as the system user, but using a normal KerberosToken) needs
                // to have the ability to read, write and alter the trace table
                conn.securityOperations().grantTablePermission(systemUser.getPrincipal(), traceTable, TablePermission.READ);
                conn.securityOperations().grantTablePermission(systemUser.getPrincipal(), traceTable, TablePermission.WRITE);
                conn.securityOperations().grantTablePermission(systemUser.getPrincipal(), traceTable, TablePermission.ALTER_TABLE);
            }
            break;
        default:
    }
}

Example 34 with MiniAccumuloClusterImpl

use of org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl in project accumulo-examples by apache.

the class ExamplesIT method testTrace.

@Test
public void testTrace() throws Exception {
    Process trace = null;
    if (ClusterType.MINI == getClusterType()) {
        MiniAccumuloClusterImpl impl = (MiniAccumuloClusterImpl) cluster;
        trace = impl.exec(TraceServer.class);
        while (!c.tableOperations().exists("trace")) sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
    }
    String[] args = new String[] { "-c", getConnectionFile(), "--createtable", "--deletetable", "--create" };
    Entry<Integer, String> pair = cluster.getClusterControl().execWithStdout(TracingExample.class, args);
    Assert.assertEquals("Expected return code of zero. STDOUT=" + pair.getValue(), 0, pair.getKey().intValue());
    String result = pair.getValue();
    Pattern pattern = Pattern.compile("TraceID: ([0-9a-f]+)");
    Matcher matcher = pattern.matcher(result);
    int count = 0;
    while (matcher.find()) {
        args = new String[] { "-c", getConnectionFile(), "--traceid", matcher.group(1) };
        pair = cluster.getClusterControl().execWithStdout(TraceDumpExample.class, args);
        count++;
    }
    assertTrue(count > 0);
    assertTrue("Output did not contain myApp@myHost", pair.getValue().contains("myApp@myHost"));
    if (ClusterType.MINI == getClusterType() && null != trace) {
        trace.destroy();
    }
}