Examples with MiniAccumuloClusterImpl org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl used on opensource projects

Example 16 with MiniAccumuloClusterImpl

use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.

the class ShellConfigIT method experimentalPropTest.

@Test
public void experimentalPropTest() throws Exception {
    // ensure experimental props do not show up in config output unless set
    AuthenticationToken token = getAdminToken();
    File clientPropsFile = null;
    switch(getClusterType()) {
        case MINI:
            MiniAccumuloClusterImpl mac = (MiniAccumuloClusterImpl) getCluster();
            clientPropsFile = mac.getConfig().getClientPropsFile();
            break;
        case STANDALONE:
            StandaloneAccumuloClusterConfiguration standaloneConf = (StandaloneAccumuloClusterConfiguration) getClusterConfiguration();
            clientPropsFile = standaloneConf.getClientPropsFile();
            break;
        default:
            fail("Unknown cluster type");
    }
    assertNotNull(clientPropsFile);
    MockShell ts = null;
    if (token instanceof PasswordToken) {
        String passwd = new String(((PasswordToken) token).getPassword(), UTF_8);
        ts = new MockShell(getAdminPrincipal(), passwd, getCluster().getInstanceName(), getCluster().getZooKeepers(), clientPropsFile);
    } else if (token instanceof KerberosToken) {
        ts = new MockShell(getAdminPrincipal(), null, getCluster().getInstanceName(), getCluster().getZooKeepers(), clientPropsFile);
    } else {
        fail("Unknown token type");
    }
    assertTrue(Property.INSTANCE_CRYPTO_PREFIX.isExperimental());
    assertTrue(Property.INSTANCE_CRYPTO_SERVICE.isExperimental());
    String configOutput = ts.exec("config");
    assertTrue(configOutput.contains(PERTABLE_CHOOSER_PROP));
    assertFalse(configOutput.contains(Property.INSTANCE_CRYPTO_SERVICE.getKey()));
}

Example 17 with MiniAccumuloClusterImpl

use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.

the class ThriftServerBindsBeforeZooKeeperLockIT method testManagerService.

@SuppressFBWarnings(value = "UNENCRYPTED_SOCKET", justification = "unencrypted socket is okay for testing")
@Test
public void testManagerService() throws Exception {
    final MiniAccumuloClusterImpl cluster = (MiniAccumuloClusterImpl) getCluster();
    try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
        final InstanceId instanceID = client.instanceOperations().getInstanceId();
        // Wait for the Manager to grab its lock
        while (true) {
            final ZooReader reader = new ZooReader(cluster.getZooKeepers(), 30000);
            try {
                List<String> locks = reader.getChildren(Constants.ZROOT + "/" + instanceID + Constants.ZMANAGER_LOCK);
                if (!locks.isEmpty()) {
                    break;
                }
            } catch (Exception e) {
                LOG.debug("Failed to find active manager location, retrying", e);
                Thread.sleep(1000);
            }
        }
        LOG.debug("Found active manager");
        int freePort = PortUtils.getRandomFreePort();
        Process manager = null;
        try {
            LOG.debug("Starting standby manager on {}", freePort);
            manager = startProcess(cluster, ServerType.MANAGER, freePort);
            while (true) {
                try (Socket s = new Socket("localhost", freePort)) {
                    if (s.isConnected()) {
                        // Pass
                        return;
                    }
                } catch (Exception e) {
                    LOG.debug("Caught exception trying to connect to Manager", e);
                }
                // Wait before trying again
                Thread.sleep(1000);
                // died trying to bind it. Pick a new port and restart it in that case.
                if (!manager.isAlive()) {
                    freePort = PortUtils.getRandomFreePort();
                    LOG.debug("Manager died, restarting it listening on {}", freePort);
                    manager = startProcess(cluster, ServerType.MANAGER, freePort);
                }
            }
        } finally {
            if (manager != null) {
                manager.destroyForcibly();
            }
        }
    }
}

Example 18 with MiniAccumuloClusterImpl

use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.

the class ThriftServerBindsBeforeZooKeeperLockIT method testMonitorService.

@SuppressFBWarnings(value = "URLCONNECTION_SSRF_FD", justification = "url is not from user")
@Test
public void testMonitorService() throws Exception {
    final MiniAccumuloClusterImpl cluster = (MiniAccumuloClusterImpl) getCluster();
    Collection<ProcessReference> monitors = cluster.getProcesses().get(ServerType.MONITOR);
    // Need to start one monitor and let it become active.
    if (monitors == null || monitors.isEmpty()) {
        getClusterControl().start(ServerType.MONITOR, "localhost");
    }
    while (true) {
        try {
            MonitorUtil.getLocation(getServerContext());
            break;
        } catch (Exception e) {
            LOG.debug("Failed to find active monitor location, retrying", e);
            Thread.sleep(1000);
        }
    }
    LOG.debug("Found active monitor");
    int freePort = PortUtils.getRandomFreePort();
    String monitorUrl = "http://localhost:" + freePort;
    Process monitor = null;
    try {
        LOG.debug("Starting standby monitor on {}", freePort);
        monitor = startProcess(cluster, ServerType.MONITOR, freePort);
        while (true) {
            URL url = new URL(monitorUrl);
            try {
                HttpURLConnection cnxn = (HttpURLConnection) url.openConnection();
                final int responseCode = cnxn.getResponseCode();
                String errorText;
                // This is our "assertion", but we want to re-check it if it's not what we expect
                if (responseCode == HttpURLConnection.HTTP_OK) {
                    return;
                } else {
                    errorText = FunctionalTestUtils.readAll(cnxn.getErrorStream());
                }
                LOG.debug("Unexpected responseCode and/or error text, will retry: '{}' '{}'", responseCode, errorText);
            } catch (Exception e) {
                LOG.debug("Caught exception trying to fetch monitor info", e);
            }
            // Wait before trying again
            Thread.sleep(1000);
            // died trying to bind it. Pick a new port and restart it in that case.
            if (!monitor.isAlive()) {
                freePort = PortUtils.getRandomFreePort();
                monitorUrl = "http://localhost:" + freePort;
                LOG.debug("Monitor died, restarting it listening on {}", freePort);
                monitor = startProcess(cluster, ServerType.MONITOR, freePort);
            }
        }
    } finally {
        if (monitor != null) {
            monitor.destroyForcibly();
        }
    }
}

Example 19 with MiniAccumuloClusterImpl

use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.

the class ThriftServerBindsBeforeZooKeeperLockIT method testGarbageCollectorPorts.

@SuppressFBWarnings(value = "UNENCRYPTED_SOCKET", justification = "unencrypted socket is okay for testing")
@Test
public void testGarbageCollectorPorts() throws Exception {
    final MiniAccumuloClusterImpl cluster = (MiniAccumuloClusterImpl) getCluster();
    try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
        InstanceId instanceID = client.instanceOperations().getInstanceId();
        // Wait for the Manager to grab its lock
        while (true) {
            final ZooReader reader = new ZooReader(cluster.getZooKeepers(), 30000);
            try {
                List<String> locks = reader.getChildren(Constants.ZROOT + "/" + instanceID + Constants.ZGC_LOCK);
                if (!locks.isEmpty()) {
                    break;
                }
            } catch (Exception e) {
                LOG.debug("Failed to find active gc location, retrying", e);
                Thread.sleep(1000);
            }
        }
        LOG.debug("Found active gc");
        int freePort = PortUtils.getRandomFreePort();
        Process manager = null;
        try {
            LOG.debug("Starting standby gc on {}", freePort);
            manager = startProcess(cluster, ServerType.GARBAGE_COLLECTOR, freePort);
            while (true) {
                try (Socket s = new Socket("localhost", freePort)) {
                    if (s.isConnected()) {
                        // Pass
                        return;
                    }
                } catch (Exception e) {
                    LOG.debug("Caught exception trying to connect to GC", e);
                }
                // Wait before trying again
                Thread.sleep(1000);
                // died trying to bind it. Pick a new port and restart it in that case.
                if (!manager.isAlive()) {
                    freePort = PortUtils.getRandomFreePort();
                    LOG.debug("GC died, restarting it listening on {}", freePort);
                    manager = startProcess(cluster, ServerType.GARBAGE_COLLECTOR, freePort);
                }
            }
        } finally {
            if (manager != null) {
                manager.destroyForcibly();
            }
        }
    }
}

Example 20 with MiniAccumuloClusterImpl

use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.

the class MultiInstanceReplicationIT method dataReplicatedToCorrectTable.

@Test
public void dataReplicatedToCorrectTable() throws Exception {
    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
    peerCfg.setNumTservers(1);
    peerCfg.setInstanceName("peer");
    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
    MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
    peer1Cluster.start();
    try (AccumuloClient clientManager = Accumulo.newClient().from(getClientProperties()).build();
        AccumuloClient clientPeer = peer1Cluster.createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD))) {
        String peerClusterName = "peer";
        String peerUserName = "peer", peerPassword = "foo";
        // Create local user
        clientPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
        // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
        String managerTable1 = "manager1", peerTable1 = "peer1", managerTable2 = "manager2", peerTable2 = "peer2";
        // Create tables
        clientPeer.tableOperations().create(peerTable1, new NewTableConfiguration());
        String peerTableId1 = clientPeer.tableOperations().tableIdMap().get(peerTable1);
        assertNotNull(peerTableId1);
        clientPeer.tableOperations().create(peerTable2, new NewTableConfiguration());
        String peerTableId2 = clientPeer.tableOperations().tableIdMap().get(peerTable2);
        assertNotNull(peerTableId2);
        Map<String, String> props1 = new HashMap<>();
        props1.put(Property.TABLE_REPLICATION.getKey(), "true");
        props1.put(Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
        clientManager.tableOperations().create(managerTable1, new NewTableConfiguration().setProperties(props1));
        String managerTableId1 = clientManager.tableOperations().tableIdMap().get(managerTable1);
        assertNotNull(managerTableId1);
        Map<String, String> props2 = new HashMap<>();
        props2.put(Property.TABLE_REPLICATION.getKey(), "true");
        props2.put(Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
        clientManager.tableOperations().create(managerTable2, new NewTableConfiguration().setProperties(props2));
        String managerTableId2 = clientManager.tableOperations().tableIdMap().get(managerTable2);
        assertNotNull(managerTableId2);
        // Give our replication user the ability to write to the tables
        clientPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
        clientPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
        // Write some data to table1
        long managerTable1Records = 0L;
        try (BatchWriter bw = clientManager.createBatchWriter(managerTable1)) {
            for (int rows = 0; rows < 2500; rows++) {
                Mutation m = new Mutation(managerTable1 + rows);
                for (int cols = 0; cols < 100; cols++) {
                    String value = Integer.toString(cols);
                    m.put(value, "", value);
                    managerTable1Records++;
                }
                bw.addMutation(m);
            }
        }
        // Write some data to table2
        long managerTable2Records = 0L;
        try (BatchWriter bw = clientManager.createBatchWriter(managerTable2)) {
            for (int rows = 0; rows < 2500; rows++) {
                Mutation m = new Mutation(managerTable2 + rows);
                for (int cols = 0; cols < 100; cols++) {
                    String value = Integer.toString(cols);
                    m.put(value, "", value);
                    managerTable2Records++;
                }
                bw.addMutation(m);
            }
        }
        log.info("Wrote all data to manager cluster");
        Set<String> filesFor1 = clientManager.replicationOperations().referencedFiles(managerTable1), filesFor2 = clientManager.replicationOperations().referencedFiles(managerTable2);
        log.info("Files to replicate for table1: " + filesFor1);
        log.info("Files to replicate for table2: " + filesFor2);
        // Restart the tserver to force a close on the WAL
        for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        cluster.exec(TabletServer.class);
        log.info("Restarted the tserver");
        // Read the data -- the tserver is back up and running
        Iterators.size(clientManager.createScanner(managerTable1, Authorizations.EMPTY).iterator());
        while (!ReplicationTable.isOnline(clientManager)) {
            log.info("Replication table still offline, waiting");
            Thread.sleep(5000);
        }
        // Wait for both tables to be replicated
        log.info("Waiting for {} for {}", filesFor1, managerTable1);
        clientManager.replicationOperations().drain(managerTable1, filesFor1);
        log.info("Waiting for {} for {}", filesFor2, managerTable2);
        clientManager.replicationOperations().drain(managerTable2, filesFor2);
        long countTable = 0L;
        try (var scanner = clientPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
            for (Entry<Key, Value> entry : scanner) {
                countTable++;
                assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(managerTable1));
            }
        }
        log.info("Found {} records in {}", countTable, peerTable1);
        assertEquals(managerTable1Records, countTable);
        countTable = 0L;
        try (var scanner = clientPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
            for (Entry<Key, Value> entry : scanner) {
                countTable++;
                assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(managerTable2));
            }
        }
        log.info("Found {} records in {}", countTable, peerTable2);
        assertEquals(managerTable2Records, countTable);
    } finally {
        peer1Cluster.stop();
    }
}