Examples with AccumuloClient org.apache.accumulo.core.client.AccumuloClient used on opensource projects

Example 46 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class GarbageCollectorCommunicatesWithTServersIT method testUnreferencedWalInTserverIsClosed.

@Test
public void testUnreferencedWalInTserverIsClosed() throws Exception {
    final String[] names = getUniqueNames(2);
    // `table` will be replicated, `otherTable` is only used to roll the WAL on the tserver
    final String table = names[0], otherTable = names[1];
    final AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build();
    // Bring the replication table online first and foremost
    ReplicationTable.setOnline(client);
    log.info("Creating {}", table);
    client.tableOperations().create(table);
    client.tableOperations().setProperty(table, Property.TABLE_REPLICATION.getKey(), "true");
    log.info("Writing a few mutations to the table");
    byte[] empty = new byte[0];
    try (BatchWriter bw = client.createBatchWriter(table)) {
        for (int i = 0; i < 5; i++) {
            Mutation m = new Mutation(Integer.toString(i));
            m.put(empty, empty, empty);
            bw.addMutation(m);
        }
        log.info("Flushing mutations to the server");
    }
    log.info("Checking that metadata only has one WAL recorded for this table");
    Set<String> wals = getWalsForTable(table);
    assertEquals("Expected to only find two WAL for the table", 2, wals.size());
    log.info("Compacting the table which will remove all WALs from the tablets");
    // Flush our test table to remove the WAL references in it
    client.tableOperations().flush(table, null, null, true);
    // Flush the metadata table too because it will have a reference to the WAL
    client.tableOperations().flush(MetadataTable.NAME, null, null, true);
    log.info("Fetching replication statuses from metadata table");
    Map<String, Status> fileToStatus = getMetadataStatusForTable(table);
    assertEquals("Expected to only find one replication status message", 1, fileToStatus.size());
    String walName = fileToStatus.keySet().iterator().next();
    assertTrue("Expected log file name from tablet to equal replication entry", wals.contains(walName));
    Status status = fileToStatus.get(walName);
    assertFalse("Expected Status for file to not be closed", status.getClosed());
    Set<String> filesForTable = getFilesForTable(table);
    assertEquals("Expected to only find one rfile for table", 1, filesForTable.size());
    log.info("Files for table before MajC: {}", filesForTable);
    // Issue a MajC to roll a new file in HDFS
    client.tableOperations().compact(table, null, null, false, true);
    Set<String> filesForTableAfterCompaction = getFilesForTable(table);
    log.info("Files for table after MajC: {}", filesForTableAfterCompaction);
    assertEquals("Expected to only find one rfile for table", 1, filesForTableAfterCompaction.size());
    assertNotEquals("Expected the files before and after compaction to differ", filesForTableAfterCompaction, filesForTable);
    // Use the rfile which was just replaced by the MajC to determine when the GC has ran
    Path fileToBeDeleted = new Path(filesForTable.iterator().next());
    FileSystem fs = getCluster().getFileSystem();
    boolean fileExists = fs.exists(fileToBeDeleted);
    while (fileExists) {
        log.info("File which should get deleted still exists: {}", fileToBeDeleted);
        Thread.sleep(2000);
        fileExists = fs.exists(fileToBeDeleted);
    }
    // At this point in time, we *know* that the GarbageCollector has run which means that the
    // Status
    // for our WAL should not be altered.
    Map<String, Status> fileToStatusAfterMinc = getMetadataStatusForTable(table);
    assertEquals("Expected to still find only one replication status message: " + fileToStatusAfterMinc, 1, fileToStatusAfterMinc.size());
    /*
     * To verify that the WALs is still getting closed, we have to force the tserver to close the
     * existing WAL and open a new one instead. The easiest way to do this is to write a load of
     * data that will exceed the 1.33% full threshold that the logger keeps track of
     */
    client.tableOperations().create(otherTable);
    try (BatchWriter bw = client.createBatchWriter(otherTable)) {
        // 500k
        byte[] bigValue = new byte[1024 * 500];
        Arrays.fill(bigValue, (byte) 1);
        // 500k * 50
        for (int i = 0; i < 50; i++) {
            Mutation m = new Mutation(Integer.toString(i));
            m.put(empty, empty, bigValue);
            bw.addMutation(m);
            if (i % 10 == 0) {
                bw.flush();
            }
        }
    }
    client.tableOperations().flush(otherTable, null, null, true);
    // Get the tservers which the manager deems as active
    final ClientContext context = (ClientContext) client;
    List<String> tservers = ManagerClient.execute(context, cli -> cli.getActiveTservers(TraceUtil.traceInfo(), context.rpcCreds()));
    assertEquals("Expected only one active tservers", 1, tservers.size());
    HostAndPort tserver = HostAndPort.fromString(tservers.get(0));
    // Get the active WALs from that server
    log.info("Fetching active WALs from {}", tserver);
    Client cli = ThriftUtil.getTServerClient(tserver, context);
    List<String> activeWalsForTserver = cli.getActiveLogs(TraceUtil.traceInfo(), context.rpcCreds());
    log.info("Active wals: {}", activeWalsForTserver);
    assertEquals("Expected to find only one active WAL", 1, activeWalsForTserver.size());
    String activeWal = new Path(activeWalsForTserver.get(0)).toString();
    assertNotEquals("Current active WAL on tserver should not be the original WAL we saw", walName, activeWal);
    log.info("Ensuring that replication status does get closed after WAL is no" + " longer in use by Tserver");
    do {
        Map<String, Status> replicationStatuses = getMetadataStatusForTable(table);
        log.info("Got replication status messages {}", replicationStatuses);
        assertEquals("Did not expect to find additional status records", 1, replicationStatuses.size());
        status = replicationStatuses.values().iterator().next();
        log.info("Current status: {}", ProtobufUtil.toString(status));
        if (status.getClosed()) {
            return;
        }
        log.info("Status is not yet closed, waiting for garbage collector to close it");
        Thread.sleep(2000);
    } while (true);
}

Example 47 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class MultiTserverReplicationIT method managerReplicationServicePortsAreAdvertised.

@Test
public void managerReplicationServicePortsAreAdvertised() throws Exception {
    // Wait for the cluster to be up
    AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build();
    ClientContext context = (ClientContext) client;
    // Wait for a tserver to come up to fulfill this request
    client.tableOperations().create("foo");
    try (Scanner s = client.createScanner("foo", Authorizations.EMPTY)) {
        assertEquals(0, Iterables.size(s));
        ZooReader zreader = new ZooReader(context.getZooKeepers(), context.getZooKeepersSessionTimeOut());
        // Should have one manager instance
        assertEquals(1, context.getManagerLocations().size());
        // Get the manager thrift service addr
        String managerAddr = Iterables.getOnlyElement(context.getManagerLocations());
        // Get the manager replication coordinator addr
        String replCoordAddr = new String(zreader.getData(ZooUtil.getRoot(client.instanceOperations().getInstanceId()) + Constants.ZMANAGER_REPLICATION_COORDINATOR_ADDR), UTF_8);
        // They shouldn't be the same
        assertNotEquals(managerAddr, replCoordAddr);
        // Neither should be zero as the port
        assertNotEquals(0, HostAndPort.fromString(managerAddr).getPort());
        assertNotEquals(0, HostAndPort.fromString(replCoordAddr).getPort());
    }
}

Example 48 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class MultiTserverReplicationIT method tserverReplicationServicePortsAreAdvertised.

@Test
public void tserverReplicationServicePortsAreAdvertised() throws Exception {
    // Wait for the cluster to be up
    AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build();
    ClientContext context = (ClientContext) client;
    // Wait for a tserver to come up to fulfill this request
    client.tableOperations().create("foo");
    try (Scanner s = client.createScanner("foo", Authorizations.EMPTY)) {
        assertEquals(0, Iterables.size(s));
        ZooReader zreader = new ZooReader(context.getZooKeepers(), context.getZooKeepersSessionTimeOut());
        Set<String> tserverHost = new HashSet<>();
        tserverHost.addAll(zreader.getChildren(ZooUtil.getRoot(client.instanceOperations().getInstanceId()) + Constants.ZTSERVERS));
        Set<HostAndPort> replicationServices = new HashSet<>();
        for (String tserver : tserverHost) {
            try {
                byte[] portData = zreader.getData(ZooUtil.getRoot(client.instanceOperations().getInstanceId()) + ReplicationConstants.ZOO_TSERVERS + "/" + tserver);
                HostAndPort replAddress = HostAndPort.fromString(new String(portData, UTF_8));
                replicationServices.add(replAddress);
            } catch (Exception e) {
                log.error("Could not find port for {}", tserver, e);
                fail("Did not find replication port advertisement for " + tserver);
            }
        }
        // Each tserver should also have equal replication services running internally
        assertEquals("Expected an equal number of replication servicers and tservers", tserverHost.size(), replicationServices.size());
    }
}

Example 49 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class ReplicationIT method replicationTableCreated.

@Test
public void replicationTableCreated() {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        assertTrue(client.tableOperations().exists(ReplicationTable.NAME));
        assertEquals(ReplicationTable.ID.canonical(), client.tableOperations().tableIdMap().get(ReplicationTable.NAME));
    }
}

Example 50 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class ReplicationIT method twoEntriesForTwoTables.

@Test
public void twoEntriesForTwoTables() throws Exception {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        String table1 = "table1", table2 = "table2";
        // replication shouldn't exist when we begin
        assertFalse("Replication table already online at the beginning of the test", ReplicationTable.isOnline(client));
        // Create two tables
        client.tableOperations().create(table1);
        client.tableOperations().create(table2);
        client.securityOperations().grantTablePermission("root", ReplicationTable.NAME, TablePermission.READ);
        // wait for permission to propagate
        Thread.sleep(5000);
        // Enable replication on table1
        client.tableOperations().setProperty(table1, Property.TABLE_REPLICATION.getKey(), "true");
        // Despite having replication on, we shouldn't have any need to write a record to it (and
        // bring
        // it online)
        assertFalse(ReplicationTable.isOnline(client));
        // Write some data to table1
        writeSomeData(client, table1, 50, 50);
        // After writing data, we'll get a replication table online
        while (!ReplicationTable.isOnline(client)) {
            sleepUninterruptibly(MILLIS_BETWEEN_REPLICATION_TABLE_ONLINE_CHECKS, TimeUnit.MILLISECONDS);
        }
        assertTrue(ReplicationTable.isOnline(client));
        // Verify that we found a single replication record that's for table1
        Entry<Key, Value> entry;
        try (Scanner s = ReplicationTable.getScanner(client)) {
            StatusSection.limit(s);
            for (int i = 0; i < 5; i++) {
                if (Iterators.size(s.iterator()) == 1) {
                    break;
                }
                Thread.sleep(1000);
            }
            entry = Iterators.getOnlyElement(s.iterator());
        }
        // We should at least find one status record for this table, we might find a second if another
        // log was started from ingesting the data
        assertEquals("Expected to find replication entry for " + table1, client.tableOperations().tableIdMap().get(table1), entry.getKey().getColumnQualifier().toString());
        // Enable replication on table2
        client.tableOperations().setProperty(table2, Property.TABLE_REPLICATION.getKey(), "true");
        // Write some data to table2
        writeSomeData(client, table2, 50, 50);
        // After the commit on these mutations, we'll get a replication entry in accumulo.metadata for
        // table2
        // Don't want to compact table2 as it ultimately cause the entry in accumulo.metadata to be
        // removed before we can verify it's there
        Set<String> tableIds = Sets.newHashSet(client.tableOperations().tableIdMap().get(table1), client.tableOperations().tableIdMap().get(table2));
        Set<String> tableIdsForMetadata = Sets.newHashSet(tableIds);
        List<Entry<Key, Value>> records = new ArrayList<>();
        try (Scanner s = client.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            s.setRange(ReplicationSection.getRange());
            for (Entry<Key, Value> metadata : s) {
                records.add(metadata);
                log.debug("Meta: {} => {}", metadata.getKey().toStringNoTruncate(), metadata.getValue());
            }
            assertEquals("Expected to find 2 records, but actually found " + records, 2, records.size());
            for (Entry<Key, Value> metadata : records) {
                assertTrue("Expected record to be in metadata but wasn't " + metadata.getKey().toStringNoTruncate() + ", tableIds remaining " + tableIdsForMetadata, tableIdsForMetadata.remove(metadata.getKey().getColumnQualifier().toString()));
            }
            assertTrue("Expected that we had removed all metadata entries " + tableIdsForMetadata, tableIdsForMetadata.isEmpty());
            // Should be creating these records in replication table from metadata table every second
            Thread.sleep(5000);
        }
        // Verify that we found two replication records: one for table1 and one for table2
        try (Scanner s = ReplicationTable.getScanner(client)) {
            StatusSection.limit(s);
            Iterator<Entry<Key, Value>> iter = s.iterator();
            assertTrue("Found no records in replication table", iter.hasNext());
            entry = iter.next();
            assertTrue("Expected to find element in replication table", tableIds.remove(entry.getKey().getColumnQualifier().toString()));
            assertTrue("Expected to find two elements in replication table, only found one ", iter.hasNext());
            entry = iter.next();
            assertTrue("Expected to find element in replication table", tableIds.remove(entry.getKey().getColumnQualifier().toString()));
            assertFalse("Expected to only find two elements in replication table", iter.hasNext());
        }
    }
}