Examples with MiniAccumuloClusterImpl org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl used on opensource projects

Example 21 with MiniAccumuloClusterImpl

use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.

the class MultiInstanceReplicationIT method dataWasReplicatedToThePeerWithoutDrain.

@Test
public void dataWasReplicatedToThePeerWithoutDrain() throws Exception {
    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
    peerCfg.setNumTservers(1);
    peerCfg.setInstanceName("peer");
    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
    MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
    peerCluster.start();
    try (AccumuloClient clientManager = Accumulo.newClient().from(getClientProperties()).build();
        AccumuloClient clientPeer = peerCluster.createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD))) {
        String peerUserName = "repl";
        String peerPassword = "passwd";
        // Create a user on the peer for replication to use
        clientPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
        String peerClusterName = "peer";
        // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
        // Configure the credentials we should use to authenticate ourselves to the peer for
        // replication
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
        String managerTable = "manager", peerTable = "peer";
        clientPeer.tableOperations().create(peerTable, new NewTableConfiguration());
        String peerTableId = clientPeer.tableOperations().tableIdMap().get(peerTable);
        assertNotNull(peerTableId);
        // Give our replication user the ability to write to the table
        clientPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
        Map<String, String> props = new HashMap<>();
        props.put(Property.TABLE_REPLICATION.getKey(), "true");
        // Replicate this table to the peerClusterName in a table with the peerTableId table id
        props.put(Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
        clientManager.tableOperations().create(managerTable, new NewTableConfiguration().setProperties(props));
        String managerTableId = clientManager.tableOperations().tableIdMap().get(managerTable);
        assertNotNull(managerTableId);
        // Write some data to table1
        try (BatchWriter bw = clientManager.createBatchWriter(managerTable)) {
            for (int rows = 0; rows < 5000; rows++) {
                Mutation m = new Mutation(Integer.toString(rows));
                for (int cols = 0; cols < 100; cols++) {
                    String value = Integer.toString(cols);
                    m.put(value, "", value);
                }
                bw.addMutation(m);
            }
        }
        log.info("Wrote all data to manager cluster");
        Set<String> files = clientManager.replicationOperations().referencedFiles(managerTable);
        log.info("Files to replicate:" + files);
        for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        cluster.exec(TabletServer.class);
        while (!ReplicationTable.isOnline(clientManager)) {
            log.info("Replication table still offline, waiting");
            Thread.sleep(5000);
        }
        Iterators.size(clientManager.createScanner(managerTable, Authorizations.EMPTY).iterator());
        try (var scanner = ReplicationTable.getScanner(clientManager)) {
            for (Entry<Key, Value> kv : scanner) {
                log.debug("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
            }
        }
        clientManager.replicationOperations().drain(managerTable, files);
        try (Scanner manager = clientManager.createScanner(managerTable, Authorizations.EMPTY);
            Scanner peer = clientPeer.createScanner(peerTable, Authorizations.EMPTY)) {
            Iterator<Entry<Key, Value>> managerIter = manager.iterator(), peerIter = peer.iterator();
            while (managerIter.hasNext() && peerIter.hasNext()) {
                Entry<Key, Value> managerEntry = managerIter.next(), peerEntry = peerIter.next();
                assertEquals(peerEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0, managerEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
                assertEquals(managerEntry.getValue(), peerEntry.getValue());
            }
            assertFalse("Had more data to read from the manager", managerIter.hasNext());
            assertFalse("Had more data to read from the peer", peerIter.hasNext());
        }
    } finally {
        peerCluster.stop();
    }
}

Example 22 with MiniAccumuloClusterImpl

use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.

the class CyclicReplicationIT method dataIsNotOverReplicated.

@Test
public void dataIsNotOverReplicated() throws Exception {
    File manager1Dir = createTestDir("manager1"), manager2Dir = createTestDir("manager2");
    String password = "password";
    MiniAccumuloConfigImpl manager1Cfg;
    MiniAccumuloClusterImpl manager1Cluster;
    while (true) {
        manager1Cfg = new MiniAccumuloConfigImpl(manager1Dir, password);
        manager1Cfg.setNumTservers(1);
        manager1Cfg.setInstanceName("manager1");
        // Set up SSL if needed
        ConfigurableMacBase.configureForEnvironment(manager1Cfg, ConfigurableMacBase.getSslDir(manager1Dir));
        manager1Cfg.setProperty(Property.REPLICATION_NAME, manager1Cfg.getInstanceName());
        manager1Cfg.setProperty(Property.TSERV_WAL_MAX_SIZE, "5M");
        manager1Cfg.setProperty(Property.REPLICATION_THREADCHECK, "5m");
        manager1Cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "1s");
        manager1Cfg.setProperty(Property.MANAGER_REPLICATION_SCAN_INTERVAL, "1s");
        manager1Cluster = new MiniAccumuloClusterImpl(manager1Cfg);
        setCoreSite(manager1Cluster);
        try {
            manager1Cluster.start();
            break;
        } catch (ZooKeeperBindException e) {
            log.warn("Failed to start ZooKeeper on {}, will retry", manager1Cfg.getZooKeeperPort());
        }
    }
    MiniAccumuloConfigImpl manager2Cfg;
    MiniAccumuloClusterImpl manager2Cluster;
    while (true) {
        manager2Cfg = new MiniAccumuloConfigImpl(manager2Dir, password);
        manager2Cfg.setNumTservers(1);
        manager2Cfg.setInstanceName("manager2");
        // Set up SSL if needed. Need to share the same SSL truststore as manager1
        this.updatePeerConfigFromPrimary(manager1Cfg, manager2Cfg);
        manager2Cfg.setProperty(Property.REPLICATION_NAME, manager2Cfg.getInstanceName());
        manager2Cfg.setProperty(Property.TSERV_WAL_MAX_SIZE, "5M");
        manager2Cfg.setProperty(Property.REPLICATION_THREADCHECK, "5m");
        manager2Cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "1s");
        manager2Cfg.setProperty(Property.MANAGER_REPLICATION_SCAN_INTERVAL, "1s");
        manager2Cluster = new MiniAccumuloClusterImpl(manager2Cfg);
        setCoreSite(manager2Cluster);
        try {
            manager2Cluster.start();
            break;
        } catch (ZooKeeperBindException e) {
            log.warn("Failed to start ZooKeeper on {}, will retry", manager2Cfg.getZooKeeperPort());
        }
    }
    try {
        AccumuloClient clientManager1 = manager1Cluster.createAccumuloClient("root", new PasswordToken(password)), clientManager2 = manager2Cluster.createAccumuloClient("root", new PasswordToken(password));
        String manager1UserName = "manager1", manager1Password = "foo";
        String manager2UserName = "manager2", manager2Password = "bar";
        String manager1Table = manager1Cluster.getInstanceName(), manager2Table = manager2Cluster.getInstanceName();
        clientManager1.securityOperations().createLocalUser(manager1UserName, new PasswordToken(manager1Password));
        clientManager2.securityOperations().createLocalUser(manager2UserName, new PasswordToken(manager2Password));
        // Configure the credentials we should use to authenticate ourselves to the peer for
        // replication
        clientManager1.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + manager2Cluster.getInstanceName(), manager2UserName);
        clientManager1.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + manager2Cluster.getInstanceName(), manager2Password);
        clientManager2.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + manager1Cluster.getInstanceName(), manager1UserName);
        clientManager2.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + manager1Cluster.getInstanceName(), manager1Password);
        clientManager1.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + manager2Cluster.getInstanceName(), ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(manager2Cluster.getInstanceName(), manager2Cluster.getZooKeepers())));
        clientManager2.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + manager1Cluster.getInstanceName(), ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(manager1Cluster.getInstanceName(), manager1Cluster.getZooKeepers())));
        clientManager1.tableOperations().create(manager1Table, new NewTableConfiguration().withoutDefaultIterators());
        String manager1TableId = clientManager1.tableOperations().tableIdMap().get(manager1Table);
        assertNotNull(manager1TableId);
        clientManager2.tableOperations().create(manager2Table, new NewTableConfiguration().withoutDefaultIterators());
        String manager2TableId = clientManager2.tableOperations().tableIdMap().get(manager2Table);
        assertNotNull(manager2TableId);
        // Replicate manager1 in the manager1 cluster to manager2 in the manager2 cluster
        clientManager1.tableOperations().setProperty(manager1Table, Property.TABLE_REPLICATION.getKey(), "true");
        clientManager1.tableOperations().setProperty(manager1Table, Property.TABLE_REPLICATION_TARGET.getKey() + manager2Cluster.getInstanceName(), manager2TableId);
        // Replicate manager2 in the manager2 cluster to manager1 in the manager2 cluster
        clientManager2.tableOperations().setProperty(manager2Table, Property.TABLE_REPLICATION.getKey(), "true");
        clientManager2.tableOperations().setProperty(manager2Table, Property.TABLE_REPLICATION_TARGET.getKey() + manager1Cluster.getInstanceName(), manager1TableId);
        // Give our replication user the ability to write to the respective table
        clientManager1.securityOperations().grantTablePermission(manager1UserName, manager1Table, TablePermission.WRITE);
        clientManager2.securityOperations().grantTablePermission(manager2UserName, manager2Table, TablePermission.WRITE);
        IteratorSetting summingCombiner = new IteratorSetting(50, SummingCombiner.class);
        SummingCombiner.setEncodingType(summingCombiner, Type.STRING);
        SummingCombiner.setCombineAllColumns(summingCombiner, true);
        // Set a combiner on both instances that will sum multiple values
        // We can use this to verify that the mutation was not sent multiple times
        clientManager1.tableOperations().attachIterator(manager1Table, summingCombiner);
        clientManager2.tableOperations().attachIterator(manager2Table, summingCombiner);
        // Write a single entry
        try (BatchWriter bw = clientManager1.createBatchWriter(manager1Table)) {
            Mutation m = new Mutation("row");
            m.put("count", "", "1");
            bw.addMutation(m);
        }
        Set<String> files = clientManager1.replicationOperations().referencedFiles(manager1Table);
        log.info("Found {} that need replication from manager1", files);
        // Kill and restart the tserver to close the WAL on manager1
        for (ProcessReference proc : manager1Cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            manager1Cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        manager1Cluster.exec(TabletServer.class);
        log.info("Restarted tserver on manager1");
        // Try to avoid ACCUMULO-2964
        Thread.sleep(1000);
        // Sanity check that the element is there on manager1
        Entry<Key, Value> entry;
        try (Scanner s = clientManager1.createScanner(manager1Table, Authorizations.EMPTY)) {
            entry = Iterables.getOnlyElement(s);
            assertEquals("1", entry.getValue().toString());
            // Wait for this table to replicate
            clientManager1.replicationOperations().drain(manager1Table, files);
            Thread.sleep(5000);
        }
        // Check that the element made it to manager2 only once
        try (Scanner s = clientManager2.createScanner(manager2Table, Authorizations.EMPTY)) {
            entry = Iterables.getOnlyElement(s);
            assertEquals("1", entry.getValue().toString());
            // Wait for manager2 to finish replicating it back
            files = clientManager2.replicationOperations().referencedFiles(manager2Table);
            // Kill and restart the tserver to close the WAL on manager2
            for (ProcessReference proc : manager2Cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
                manager2Cluster.killProcess(ServerType.TABLET_SERVER, proc);
            }
            manager2Cluster.exec(TabletServer.class);
            // Try to avoid ACCUMULO-2964
            Thread.sleep(1000);
        }
        // Check that the element made it to manager2 only once
        try (Scanner s = clientManager2.createScanner(manager2Table, Authorizations.EMPTY)) {
            entry = Iterables.getOnlyElement(s);
            assertEquals("1", entry.getValue().toString());
            clientManager2.replicationOperations().drain(manager2Table, files);
            Thread.sleep(5000);
        }
        // Verify that the entry wasn't sent back to manager1
        try (Scanner s = clientManager1.createScanner(manager1Table, Authorizations.EMPTY)) {
            entry = Iterables.getOnlyElement(s);
            assertEquals("1", entry.getValue().toString());
        }
    } finally {
        manager1Cluster.stop();
        manager2Cluster.stop();
    }
}

Example 23 with MiniAccumuloClusterImpl

use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.

the class ExternalCompactionMetricsIT method testMetrics.

@Test
public void testMetrics() throws Exception {
    Collection<ProcessReference> tservers = ((MiniAccumuloClusterImpl) getCluster()).getProcesses().get(ServerType.TABLET_SERVER);
    assertEquals(2, tservers.size());
    // kill one tserver so that queue metrics are not spread across tservers
    ((MiniAccumuloClusterImpl) getCluster()).killProcess(TABLET_SERVER, tservers.iterator().next());
    String[] names = getUniqueNames(2);
    try (final AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
        String table1 = names[0];
        createTable(client, table1, "cs1", 5);
        String table2 = names[1];
        createTable(client, table2, "cs2", 10);
        writeData(client, table1);
        writeData(client, table2);
        final LinkedBlockingQueue<Metric> queueMetrics = new LinkedBlockingQueue<>();
        final AtomicBoolean shutdownTailer = new AtomicBoolean(false);
        Thread thread = Threads.createThread("metric-tailer", () -> {
            while (!shutdownTailer.get()) {
                List<String> statsDMetrics = sink.getLines();
                for (String s : statsDMetrics) {
                    if (shutdownTailer.get()) {
                        break;
                    }
                    if (s.startsWith(MetricsProducer.METRICS_MAJC_QUEUED)) {
                        queueMetrics.add(TestStatsDSink.parseStatsDMetric(s));
                    }
                }
            }
        });
        thread.start();
        compact(client, table1, 7, "DCQ1", false);
        compact(client, table2, 13, "DCQ2", false);
        boolean sawDCQ1_5 = false;
        boolean sawDCQ2_10 = false;
        // wait until expected number of queued are seen in metrics
        while (!sawDCQ1_5 || !sawDCQ2_10) {
            Metric qm = queueMetrics.take();
            sawDCQ1_5 |= match(qm, "DCQ1", "5");
            sawDCQ2_10 |= match(qm, "DCQ2", "10");
        }
        cluster.getClusterControl().startCompactors(Compactor.class, 1, QUEUE1);
        cluster.getClusterControl().startCompactors(Compactor.class, 1, QUEUE2);
        cluster.getClusterControl().startCoordinator(CompactionCoordinator.class);
        boolean sawDCQ1_0 = false;
        boolean sawDCQ2_0 = false;
        // wait until queued goes to zero in metrics
        while (!sawDCQ1_0 || !sawDCQ2_0) {
            Metric qm = queueMetrics.take();
            sawDCQ1_0 |= match(qm, "DCQ1", "0");
            sawDCQ2_0 |= match(qm, "DCQ2", "0");
        }
        shutdownTailer.set(true);
        thread.join();
        // Wait for all external compactions to complete
        long count;
        do {
            UtilWaitThread.sleep(100);
            try (TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forLevel(DataLevel.USER).fetch(ColumnType.ECOMP).build()) {
                count = tm.stream().flatMap(t -> t.getExternalCompactions().keySet().stream()).count();
            }
        } while (count > 0);
        verify(client, table1, 7);
        verify(client, table2, 13);
    } finally {
    // We stopped the TServer and started our own, restart the original TabletServers
    // Uncomment this if other tests are added.
    // 
    // cluster.getClusterControl().start(ServerType.TABLET_SERVER);
    }
}

Example 24 with MiniAccumuloClusterImpl

use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.

the class ConfigurableMacBase method createMiniAccumulo.

@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "path provided by test")
private void createMiniAccumulo() throws Exception {
    // createTestDir will give us a empty directory, we don't need to clean it up ourselves
    File baseDir = createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName());
    MiniAccumuloConfigImpl cfg = new MiniAccumuloConfigImpl(baseDir, ROOT_PASSWORD);
    File nativePathInDevTree = NativeMapIT.nativeMapLocation();
    File nativePathInMapReduce = new File(System.getProperty("user.dir"));
    cfg.setNativeLibPaths(nativePathInDevTree.getAbsolutePath(), nativePathInMapReduce.toString());
    Configuration coreSite = new Configuration(false);
    cfg.setProperty(Property.TSERV_NATIVEMAP_ENABLED, Boolean.TRUE.toString());
    configure(cfg, coreSite);
    configureForEnvironment(cfg, getSslDir(baseDir));
    if (Boolean.parseBoolean(cfg.getSiteConfig().get(Property.TSERV_NATIVEMAP_ENABLED.getKey()))) {
        NativeMapLoader.loadForTest(List.of(nativePathInDevTree, nativePathInMapReduce), () -> {
            throw new IllegalStateException("Native maps were configured, but not available");
        });
    }
    cluster = new MiniAccumuloClusterImpl(cfg);
    if (coreSite.size() > 0) {
        File csFile = new File(cluster.getConfig().getConfDir(), "core-site.xml");
        if (csFile.exists()) {
            coreSite.addResource(new Path(csFile.getAbsolutePath()));
        }
        File tmp = new File(csFile.getAbsolutePath() + ".tmp");
        OutputStream out = new BufferedOutputStream(new FileOutputStream(tmp));
        coreSite.writeXml(out);
        out.close();
        assertTrue(tmp.renameTo(csFile));
    }
    beforeClusterStart(cfg);
}

Example 25 with MiniAccumuloClusterImpl

use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.

the class CloneTestIT method testOfflineClone.

@Test
public void testOfflineClone() throws Exception {
    String[] tableNames = getUniqueNames(3);
    String table1 = tableNames[0];
    String table2 = tableNames[1];
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        AccumuloCluster cluster = getCluster();
        assumeTrue(cluster instanceof MiniAccumuloClusterImpl);
        c.tableOperations().create(table1);
        writeData(table1, c);
        Map<String, String> props = new HashMap<>();
        props.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "500K");
        Set<String> exclude = new HashSet<>();
        exclude.add(Property.TABLE_FILE_MAX.getKey());
        c.tableOperations().clone(table1, table2, CloneConfiguration.builder().setFlush(true).setPropertiesToSet(props).setPropertiesToExclude(exclude).setKeepOffline(true).build());
        assertTableState(table2, c, TableState.OFFLINE);
        // delete tables
        c.tableOperations().delete(table1);
        c.tableOperations().delete(table2);
    }
}