Examples with MiniAccumuloClusterImpl org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl used on opensource projects

Example 1 with MiniAccumuloClusterImpl

use of org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl in project accumulo by apache.

the class ClassLoaderIT method checkCluster.

@Before
public void checkCluster() {
    Assume.assumeThat(getClusterType(), CoreMatchers.is(ClusterType.MINI));
    MiniAccumuloClusterImpl mac = (MiniAccumuloClusterImpl) getCluster();
    rootPath = mac.getConfig().getDir().getAbsolutePath();
}

Example 2 with MiniAccumuloClusterImpl

use of org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl in project accumulo by apache.

the class CloneTestIT method testDeleteClone.

@Test
public void testDeleteClone() throws Exception {
    String[] tableNames = getUniqueNames(3);
    String table1 = tableNames[0];
    String table2 = tableNames[1];
    String table3 = tableNames[2];
    Connector c = getConnector();
    AccumuloCluster cluster = getCluster();
    Assume.assumeTrue(cluster instanceof MiniAccumuloClusterImpl);
    MiniAccumuloClusterImpl mac = (MiniAccumuloClusterImpl) cluster;
    String rootPath = mac.getConfig().getDir().getAbsolutePath();
    // verify that deleting a new table removes the files
    c.tableOperations().create(table3);
    writeData(table3, c).close();
    c.tableOperations().flush(table3, null, null, true);
    // check for files
    FileSystem fs = getCluster().getFileSystem();
    String id = c.tableOperations().tableIdMap().get(table3);
    FileStatus[] status = fs.listStatus(new Path(rootPath + "/accumulo/tables/" + id));
    assertTrue(status.length > 0);
    // verify disk usage
    List<DiskUsage> diskUsage = c.tableOperations().getDiskUsage(Collections.singleton(table3));
    assertEquals(1, diskUsage.size());
    assertTrue(diskUsage.get(0).getUsage() > 100);
    // delete the table
    c.tableOperations().delete(table3);
    // verify its gone from the file system
    Path tablePath = new Path(rootPath + "/accumulo/tables/" + id);
    if (fs.exists(tablePath)) {
        status = fs.listStatus(tablePath);
        assertTrue(status == null || status.length == 0);
    }
    c.tableOperations().create(table1);
    BatchWriter bw = writeData(table1, c);
    Map<String, String> props = new HashMap<>();
    props.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "500K");
    Set<String> exclude = new HashSet<>();
    exclude.add(Property.TABLE_FILE_MAX.getKey());
    c.tableOperations().clone(table1, table2, true, props, exclude);
    Mutation m3 = new Mutation("009");
    m3.put("data", "x", "1");
    m3.put("data", "y", "2");
    bw.addMutation(m3);
    bw.close();
    // delete source table, should not affect clone
    c.tableOperations().delete(table1);
    checkData(table2, c);
    c.tableOperations().compact(table2, null, null, true, true);
    checkData(table2, c);
    c.tableOperations().delete(table2);
}

Example 3 with MiniAccumuloClusterImpl

use of org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl in project accumulo by apache.

the class ThriftServerBindsBeforeZooKeeperLockIT method testMasterService.

@Test
public void testMasterService() throws Exception {
    final MiniAccumuloClusterImpl cluster = (MiniAccumuloClusterImpl) getCluster();
    final ZooKeeperInstance inst = new ZooKeeperInstance(cluster.getClientConfig());
    // Wait for the Master to grab its lock
    while (true) {
        final ZooReader reader = new ZooReader(inst.getZooKeepers(), 30000);
        try {
            List<String> locks = reader.getChildren(Constants.ZROOT + "/" + inst.getInstanceID() + Constants.ZMASTER_LOCK);
            if (locks.size() > 0) {
                break;
            }
        } catch (Exception e) {
            LOG.debug("Failed to find active master location, retrying", e);
            Thread.sleep(1000);
        }
    }
    LOG.debug("Found active master");
    while (true) {
        int freePort = PortUtils.getRandomFreePort();
        Process master = null;
        try {
            LOG.debug("Starting standby master on {}", freePort);
            master = startProcess(cluster, ServerType.MASTER, freePort);
            while (true) {
                Socket s = null;
                try {
                    s = new Socket("localhost", freePort);
                    if (s.isConnected()) {
                        // Pass
                        return;
                    }
                } catch (Exception e) {
                    LOG.debug("Caught exception trying to connect to Master", e);
                } finally {
                    if (null != s) {
                        s.close();
                    }
                }
                // Wait before trying again
                Thread.sleep(1000);
                // died trying to bind it. Pick a new port and restart it in that case.
                if (!master.isAlive()) {
                    freePort = PortUtils.getRandomFreePort();
                    LOG.debug("Master died, restarting it listening on {}", freePort);
                    master = startProcess(cluster, ServerType.MASTER, freePort);
                }
            }
        } finally {
            if (null != master) {
                master.destroyForcibly();
            }
        }
    }
}

Example 4 with MiniAccumuloClusterImpl

use of org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl in project accumulo by apache.

the class TracerRecoversAfterOfflineTableIT method test.

@Test
public void test() throws Exception {
    Process tracer = null;
    Connector conn = getConnector();
    if (!conn.tableOperations().exists("trace")) {
        MiniAccumuloClusterImpl mac = cluster;
        tracer = mac.exec(TraceServer.class);
        while (!conn.tableOperations().exists("trace")) {
            sleepUninterruptibly(1, TimeUnit.SECONDS);
        }
        sleepUninterruptibly(5, TimeUnit.SECONDS);
    }
    log.info("Taking table offline");
    conn.tableOperations().offline("trace", true);
    String tableName = getUniqueNames(1)[0];
    conn.tableOperations().create(tableName);
    log.info("Start a distributed trace span");
    DistributedTrace.enable("localhost", "testTrace", getClientConfig());
    Span root = Trace.on("traceTest");
    BatchWriter bw = conn.createBatchWriter(tableName, null);
    Mutation m = new Mutation("m");
    m.put("a", "b", "c");
    bw.addMutation(m);
    bw.close();
    root.stop();
    log.info("Bringing trace table back online");
    conn.tableOperations().online("trace", true);
    log.info("Trace table is online, should be able to find trace");
    try (Scanner scanner = conn.createScanner("trace", Authorizations.EMPTY)) {
        scanner.setRange(new Range(new Text(Long.toHexString(root.traceId()))));
        while (true) {
            final StringBuilder finalBuffer = new StringBuilder();
            int traceCount = TraceDump.printTrace(scanner, new Printer() {

                @Override
                public void print(final String line) {
                    try {
                        finalBuffer.append(line).append("\n");
                    } catch (Exception ex) {
                        throw new RuntimeException(ex);
                    }
                }
            });
            String traceOutput = finalBuffer.toString();
            log.info("Trace output:{}", traceOutput);
            if (traceCount > 0) {
                int lastPos = 0;
                for (String part : "traceTest,close,binMutations".split(",")) {
                    log.info("Looking in trace output for '{}'", part);
                    int pos = traceOutput.indexOf(part);
                    assertTrue("Did not find '" + part + "' in output", pos > 0);
                    assertTrue("'" + part + "' occurred earlier than the previous element unexpectedly", pos > lastPos);
                    lastPos = pos;
                }
                break;
            } else {
                log.info("Ignoring trace output as traceCount not greater than zero: {}", traceCount);
                Thread.sleep(1000);
            }
        }
        if (tracer != null) {
            tracer.destroy();
        }
    }
}

Example 5 with MiniAccumuloClusterImpl

use of org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl in project accumulo by apache.

the class MultiInstanceReplicationIT method dataWasReplicatedToThePeer.

@Test(timeout = 10 * 60 * 1000)
public void dataWasReplicatedToThePeer() throws Exception {
    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
    peerCfg.setNumTservers(1);
    peerCfg.setInstanceName("peer");
    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
    MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
    peerCluster.start();
    try {
        final Connector connMaster = getConnector();
        final Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
        ReplicationTable.setOnline(connMaster);
        String peerUserName = "peer", peerPassword = "foo";
        String peerClusterName = "peer";
        connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
        // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
        final String masterTable = "master", peerTable = "peer";
        connMaster.tableOperations().create(masterTable);
        String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
        Assert.assertNotNull(masterTableId);
        connPeer.tableOperations().create(peerTable);
        String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
        Assert.assertNotNull(peerTableId);
        connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
        // Replicate this table to the peerClusterName in a table with the peerTableId table id
        connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
        connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
        // Write some data to table1
        BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
        for (int rows = 0; rows < 5000; rows++) {
            Mutation m = new Mutation(Integer.toString(rows));
            for (int cols = 0; cols < 100; cols++) {
                String value = Integer.toString(cols);
                m.put(value, "", value);
            }
            bw.addMutation(m);
        }
        bw.close();
        log.info("Wrote all data to master cluster");
        final Set<String> filesNeedingReplication = connMaster.replicationOperations().referencedFiles(masterTable);
        log.info("Files to replicate: " + filesNeedingReplication);
        for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        cluster.exec(TabletServer.class);
        log.info("TabletServer restarted");
        Iterators.size(ReplicationTable.getScanner(connMaster).iterator());
        log.info("TabletServer is online");
        while (!ReplicationTable.isOnline(connMaster)) {
            log.info("Replication table still offline, waiting");
            Thread.sleep(5000);
        }
        log.info("");
        log.info("Fetching metadata records:");
        for (Entry<Key, Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
            } else {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), kv.getValue());
            }
        }
        log.info("");
        log.info("Fetching replication records:");
        for (Entry<Key, Value> kv : ReplicationTable.getScanner(connMaster)) {
            log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
        }
        Future<Boolean> future = executor.submit(new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                long then = System.currentTimeMillis();
                connMaster.replicationOperations().drain(masterTable, filesNeedingReplication);
                long now = System.currentTimeMillis();
                log.info("Drain completed in " + (now - then) + "ms");
                return true;
            }
        });
        try {
            future.get(60, TimeUnit.SECONDS);
        } catch (TimeoutException e) {
            future.cancel(true);
            Assert.fail("Drain did not finish within 60 seconds");
        } finally {
            executor.shutdownNow();
        }
        log.info("drain completed");
        log.info("");
        log.info("Fetching metadata records:");
        for (Entry<Key, Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
            } else {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), kv.getValue());
            }
        }
        log.info("");
        log.info("Fetching replication records:");
        for (Entry<Key, Value> kv : ReplicationTable.getScanner(connMaster)) {
            log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
        }
        try (Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY);
            Scanner peer = connPeer.createScanner(peerTable, Authorizations.EMPTY)) {
            Iterator<Entry<Key, Value>> masterIter = master.iterator(), peerIter = peer.iterator();
            Entry<Key, Value> masterEntry = null, peerEntry = null;
            while (masterIter.hasNext() && peerIter.hasNext()) {
                masterEntry = masterIter.next();
                peerEntry = peerIter.next();
                Assert.assertEquals(masterEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0, masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
                Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
            }
            log.info("Last master entry: {}", masterEntry);
            log.info("Last peer entry: {}", peerEntry);
            Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
            Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
        }
    } finally {
        peerCluster.stop();
    }
}