Examples with MiniAccumuloClusterImpl org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl used on opensource projects

Example 11 with MiniAccumuloClusterImpl

use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.

the class ExistingMacIT method testExistingRunningInstance.

@Test
public void testExistingRunningInstance() throws Exception {
    final String table = getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        // Ensure that a manager and tserver are up so the existing instance check won't fail.
        client.tableOperations().create(table);
        try (BatchWriter bw = client.createBatchWriter(table)) {
            Mutation m = new Mutation("foo");
            m.put("cf", "cq", "value");
            bw.addMutation(m);
        }
        File hadoopConfDir = createTestDir(ExistingMacIT.class.getSimpleName() + "_hadoop_conf_2");
        FileUtils.deleteQuietly(hadoopConfDir);
        assertTrue(hadoopConfDir.mkdirs());
        createEmptyConfig(new File(hadoopConfDir, "core-site.xml"));
        createEmptyConfig(new File(hadoopConfDir, "hdfs-site.xml"));
        File testDir2 = createTestDir(ExistingMacIT.class.getSimpleName() + "_3");
        FileUtils.deleteQuietly(testDir2);
        MiniAccumuloConfigImpl macConfig2 = new MiniAccumuloConfigImpl(testDir2, "notused");
        macConfig2.useExistingInstance(new File(getCluster().getConfig().getConfDir(), "accumulo.properties"), hadoopConfDir);
        System.out.println("conf " + new File(getCluster().getConfig().getConfDir(), "accumulo.properties"));
        MiniAccumuloClusterImpl accumulo2 = new MiniAccumuloClusterImpl(macConfig2);
        RuntimeException e = assertThrows("A 2nd MAC instance should not be able to start over an existing MAC instance", RuntimeException.class, accumulo2::start);
        assertEquals("The Accumulo instance being used is already running. Aborting.", e.getMessage());
    }
}

Example 12 with MiniAccumuloClusterImpl

use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.

the class MissingWalHeaderCompletesRecoveryIT method testPartialHeaderWalRecoveryCompletes.

@Test
public void testPartialHeaderWalRecoveryCompletes() throws Exception {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        MiniAccumuloClusterImpl cluster = getCluster();
        FileSystem fs = getCluster().getFileSystem();
        // Fake out something that looks like host:port, it's irrelevant
        String fakeServer = "127.0.0.1:12345";
        File walogs = new File(cluster.getConfig().getAccumuloDir(), Constants.WAL_DIR);
        File walogServerDir = new File(walogs, fakeServer.replace(':', '+'));
        File partialHeaderWalog = new File(walogServerDir, UUID.randomUUID().toString());
        log.info("Created WAL with malformed header at {}", partialHeaderWalog.toURI());
        // Write half of the header
        FSDataOutputStream wal = fs.create(new Path(partialHeaderWalog.toURI()));
        wal.write(DfsLogger.LOG_FILE_HEADER_V4.getBytes(UTF_8), 0, DfsLogger.LOG_FILE_HEADER_V4.length() / 2);
        wal.close();
        assertTrue("root user did not have write permission to metadata table", client.securityOperations().hasTablePermission("root", MetadataTable.NAME, TablePermission.WRITE));
        String tableName = getUniqueNames(1)[0];
        client.tableOperations().create(tableName);
        TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName));
        assertNotNull("Table ID was null", tableId);
        LogEntry logEntry = new LogEntry(null, 0, partialHeaderWalog.toURI().toString());
        log.info("Taking {} offline", tableName);
        client.tableOperations().offline(tableName, true);
        log.info("{} is offline", tableName);
        Text row = TabletsSection.encodeRow(tableId, null);
        Mutation m = new Mutation(row);
        m.put(logEntry.getColumnFamily(), logEntry.getColumnQualifier(), logEntry.getValue());
        try (BatchWriter bw = client.createBatchWriter(MetadataTable.NAME)) {
            bw.addMutation(m);
        }
        log.info("Bringing {} online", tableName);
        client.tableOperations().online(tableName, true);
        log.info("{} is online", tableName);
        // otherwise the tablet will never come online and we won't be able to read it.
        try (Scanner s = client.createScanner(tableName, Authorizations.EMPTY)) {
            assertEquals(0, Iterables.size(s));
        }
    }
}

Example 13 with MiniAccumuloClusterImpl

use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.

the class MissingWalHeaderCompletesRecoveryIT method testEmptyWalRecoveryCompletes.

@Test
public void testEmptyWalRecoveryCompletes() throws Exception {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        MiniAccumuloClusterImpl cluster = getCluster();
        FileSystem fs = cluster.getFileSystem();
        // Fake out something that looks like host:port, it's irrelevant
        String fakeServer = "127.0.0.1:12345";
        File walogs = new File(cluster.getConfig().getAccumuloDir(), Constants.WAL_DIR);
        File walogServerDir = new File(walogs, fakeServer.replace(':', '+'));
        File emptyWalog = new File(walogServerDir, UUID.randomUUID().toString());
        log.info("Created empty WAL at {}", emptyWalog.toURI());
        fs.create(new Path(emptyWalog.toURI())).close();
        assertTrue("root user did not have write permission to metadata table", client.securityOperations().hasTablePermission("root", MetadataTable.NAME, TablePermission.WRITE));
        String tableName = getUniqueNames(1)[0];
        client.tableOperations().create(tableName);
        TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName));
        assertNotNull("Table ID was null", tableId);
        LogEntry logEntry = new LogEntry(new KeyExtent(tableId, null, null), 0, emptyWalog.toURI().toString());
        log.info("Taking {} offline", tableName);
        client.tableOperations().offline(tableName, true);
        log.info("{} is offline", tableName);
        Text row = TabletsSection.encodeRow(tableId, null);
        Mutation m = new Mutation(row);
        m.put(logEntry.getColumnFamily(), logEntry.getColumnQualifier(), logEntry.getValue());
        try (BatchWriter bw = client.createBatchWriter(MetadataTable.NAME)) {
            bw.addMutation(m);
        }
        log.info("Bringing {} online", tableName);
        client.tableOperations().online(tableName, true);
        log.info("{} is online", tableName);
        // otherwise the tablet will never come online and we won't be able to read it.
        try (Scanner s = client.createScanner(tableName, Authorizations.EMPTY)) {
            assertEquals(0, Iterables.size(s));
        }
    }
}

Example 14 with MiniAccumuloClusterImpl

use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.

the class UnorderedWorkAssignerReplicationIT method dataReplicatedToCorrectTable.

@Test
public void dataReplicatedToCorrectTable() throws Exception {
    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
    peerCfg.setNumTservers(1);
    peerCfg.setInstanceName("peer");
    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
    MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
    peer1Cluster.start();
    try (AccumuloClient clientManager = Accumulo.newClient().from(getClientProperties()).build();
        AccumuloClient clientPeer = peer1Cluster.createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD))) {
        String peerClusterName = "peer";
        String peerUserName = "peer", peerPassword = "foo";
        // Create local user
        clientPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
        // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
        String managerTable1 = "manager1", peerTable1 = "peer1", managerTable2 = "manager2", peerTable2 = "peer2";
        // Create tables
        clientPeer.tableOperations().create(peerTable1);
        String peerTableId1 = clientPeer.tableOperations().tableIdMap().get(peerTable1);
        assertNotNull(peerTableId1);
        clientPeer.tableOperations().create(peerTable2);
        String peerTableId2 = clientPeer.tableOperations().tableIdMap().get(peerTable2);
        assertNotNull(peerTableId2);
        // Grant write permission
        clientPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
        clientPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
        Map<String, String> props1 = new HashMap<>();
        props1.put(Property.TABLE_REPLICATION.getKey(), "true");
        props1.put(Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
        clientManager.tableOperations().create(managerTable1, new NewTableConfiguration().setProperties(props1));
        String managerTableId1 = clientManager.tableOperations().tableIdMap().get(managerTable1);
        assertNotNull(managerTableId1);
        Map<String, String> props2 = new HashMap<>();
        props2.put(Property.TABLE_REPLICATION.getKey(), "true");
        props2.put(Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
        clientManager.tableOperations().create(managerTable2, new NewTableConfiguration().setProperties(props2));
        String managerTableId2 = clientManager.tableOperations().tableIdMap().get(managerTable2);
        assertNotNull(managerTableId2);
        // Wait for zookeeper updates (configuration) to propagate
        sleepUninterruptibly(3, TimeUnit.SECONDS);
        // Write some data to table1
        long managerTable1Records = 0L;
        try (BatchWriter bw = clientManager.createBatchWriter(managerTable1)) {
            for (int rows = 0; rows < 2500; rows++) {
                Mutation m = new Mutation(managerTable1 + rows);
                for (int cols = 0; cols < 100; cols++) {
                    String value = Integer.toString(cols);
                    m.put(value, "", value);
                    managerTable1Records++;
                }
                bw.addMutation(m);
            }
        }
        // Write some data to table2
        long managerTable2Records = 0L;
        try (BatchWriter bw = clientManager.createBatchWriter(managerTable2)) {
            for (int rows = 0; rows < 2500; rows++) {
                Mutation m = new Mutation(managerTable2 + rows);
                for (int cols = 0; cols < 100; cols++) {
                    String value = Integer.toString(cols);
                    m.put(value, "", value);
                    managerTable2Records++;
                }
                bw.addMutation(m);
            }
        }
        log.info("Wrote all data to manager cluster");
        Set<String> filesFor1 = clientManager.replicationOperations().referencedFiles(managerTable1), filesFor2 = clientManager.replicationOperations().referencedFiles(managerTable2);
        while (!ReplicationTable.isOnline(clientManager)) {
            Thread.sleep(500);
        }
        // Restart the tserver to force a close on the WAL
        for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        cluster.exec(TabletServer.class);
        log.info("Restarted the tserver");
        // Read the data -- the tserver is back up and running
        Iterators.size(clientManager.createScanner(managerTable1, Authorizations.EMPTY).iterator());
        // Wait for both tables to be replicated
        log.info("Waiting for {} for {}", filesFor1, managerTable1);
        clientManager.replicationOperations().drain(managerTable1, filesFor1);
        log.info("Waiting for {} for {}", filesFor2, managerTable2);
        clientManager.replicationOperations().drain(managerTable2, filesFor2);
        long countTable = 0L;
        for (int i = 0; i < 5; i++) {
            countTable = 0L;
            for (Entry<Key, Value> entry : clientPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
                countTable++;
                assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(managerTable1));
            }
            log.info("Found {} records in {}", countTable, peerTable1);
            if (managerTable1Records != countTable) {
                log.warn("Did not find {} expected records in {}, only found {}", managerTable1Records, peerTable1, countTable);
            }
        }
        assertEquals(managerTable1Records, countTable);
        for (int i = 0; i < 5; i++) {
            countTable = 0L;
            for (Entry<Key, Value> entry : clientPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
                countTable++;
                assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(managerTable2));
            }
            log.info("Found {} records in {}", countTable, peerTable2);
            if (managerTable2Records != countTable) {
                log.warn("Did not find {} expected records in {}, only found {}", managerTable2Records, peerTable2, countTable);
            }
        }
        assertEquals(managerTable2Records, countTable);
    } finally {
        peer1Cluster.stop();
    }
}

Example 15 with MiniAccumuloClusterImpl

use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.

the class UnorderedWorkAssignerReplicationIT method dataWasReplicatedToThePeerWithoutDrain.

@Test
public void dataWasReplicatedToThePeerWithoutDrain() throws Exception {
    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
    peerCfg.setNumTservers(1);
    peerCfg.setInstanceName("peer");
    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
    MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
    peerCluster.start();
    try (AccumuloClient clientManager = Accumulo.newClient().from(getClientProperties()).build();
        AccumuloClient clientPeer = peerCluster.createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD))) {
        String peerUserName = "repl";
        String peerPassword = "passwd";
        // Create a user on the peer for replication to use
        clientPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
        String peerClusterName = "peer";
        // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
        // Configure the credentials we should use to authenticate ourselves to the peer for
        // replication
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
        clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
        String managerTable = "manager", peerTable = "peer";
        clientPeer.tableOperations().create(peerTable);
        String peerTableId = clientPeer.tableOperations().tableIdMap().get(peerTable);
        assertNotNull(peerTableId);
        Map<String, String> props = new HashMap<>();
        props.put(Property.TABLE_REPLICATION.getKey(), "true");
        props.put(Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
        clientManager.tableOperations().create(managerTable, new NewTableConfiguration().setProperties(props));
        String managerTableId = clientManager.tableOperations().tableIdMap().get(managerTable);
        assertNotNull(managerTableId);
        // Give our replication user the ability to write to the table
        clientPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
        // Write some data to table1
        try (BatchWriter bw = clientManager.createBatchWriter(managerTable)) {
            for (int rows = 0; rows < 5000; rows++) {
                Mutation m = new Mutation(Integer.toString(rows));
                for (int cols = 0; cols < 100; cols++) {
                    String value = Integer.toString(cols);
                    m.put(value, "", value);
                }
                bw.addMutation(m);
            }
        }
        log.info("Wrote all data to manager cluster");
        Set<String> files = clientManager.replicationOperations().referencedFiles(managerTable);
        for (String s : files) {
            log.info("Found referenced file for {}: {}", managerTable, s);
        }
        for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        cluster.exec(TabletServer.class);
        Iterators.size(clientManager.createScanner(managerTable, Authorizations.EMPTY).iterator());
        try (var scanner = clientManager.createScanner(ReplicationTable.NAME, Authorizations.EMPTY)) {
            for (Entry<Key, Value> kv : scanner) {
                log.debug("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
            }
        }
        clientManager.replicationOperations().drain(managerTable, files);
        try (Scanner manager = clientManager.createScanner(managerTable, Authorizations.EMPTY);
            Scanner peer = clientPeer.createScanner(peerTable, Authorizations.EMPTY)) {
            Iterator<Entry<Key, Value>> managerIter = manager.iterator(), peerIter = peer.iterator();
            assertTrue("No data in manager table", managerIter.hasNext());
            assertTrue("No data in peer table", peerIter.hasNext());
            while (managerIter.hasNext() && peerIter.hasNext()) {
                Entry<Key, Value> managerEntry = managerIter.next(), peerEntry = peerIter.next();
                assertEquals(peerEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0, managerEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
                assertEquals(managerEntry.getValue(), peerEntry.getValue());
            }
            assertFalse("Had more data to read from the manager", managerIter.hasNext());
            assertFalse("Had more data to read from the peer", peerIter.hasNext());
        }
        peerCluster.stop();
    }
}