Examples with BatchWriter org.apache.accumulo.core.client.BatchWriter used on opensource projects

Example 71 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class FinishedWorkUpdaterIT method recordsWithProgressUpdateBothTables.

@Test
public void recordsWithProgressUpdateBothTables() throws Exception {
    conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.READ);
    conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.WRITE);
    ReplicationTable.setOnline(conn);
    String file = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    Status stat = Status.newBuilder().setBegin(100).setEnd(200).setClosed(true).setInfiniteEnd(false).build();
    ReplicationTarget target = new ReplicationTarget("peer", "table1", Table.ID.of("1"));
    // Create a single work record for a file to some peer
    BatchWriter bw = ReplicationTable.getBatchWriter(conn);
    Mutation m = new Mutation(file);
    WorkSection.add(m, target.toText(), ProtobufUtil.toValue(stat));
    bw.addMutation(m);
    bw.close();
    updater.run();
    try (Scanner s = ReplicationTable.getScanner(conn)) {
        s.setRange(Range.exact(file));
        StatusSection.limit(s);
        Entry<Key, Value> entry = Iterables.getOnlyElement(s);
        Assert.assertEquals(entry.getKey().getColumnFamily(), StatusSection.NAME);
        Assert.assertEquals(entry.getKey().getColumnQualifier().toString(), target.getSourceTableId().canonicalID());
        // We should only rely on the correct begin attribute being returned
        Status actual = Status.parseFrom(entry.getValue().get());
        Assert.assertEquals(stat.getBegin(), actual.getBegin());
    }
}

Example 72 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class FinishedWorkUpdaterIT method chooseMinimumBeginOffset.

@Test
public void chooseMinimumBeginOffset() throws Exception {
    conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.READ);
    conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.WRITE);
    ReplicationTable.setOnline(conn);
    String file = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    // @formatter:off
    Status stat1 = Status.newBuilder().setBegin(100).setEnd(1000).setClosed(true).setInfiniteEnd(false).build(), stat2 = Status.newBuilder().setBegin(500).setEnd(1000).setClosed(true).setInfiniteEnd(false).build(), stat3 = Status.newBuilder().setBegin(1).setEnd(1000).setClosed(true).setInfiniteEnd(false).build();
    ReplicationTarget target1 = new ReplicationTarget("peer1", "table1", Table.ID.of("1")), target2 = new ReplicationTarget("peer2", "table2", Table.ID.of("1")), target3 = new ReplicationTarget("peer3", "table3", Table.ID.of("1"));
    // @formatter:on
    // Create a single work record for a file to some peer
    BatchWriter bw = ReplicationTable.getBatchWriter(conn);
    Mutation m = new Mutation(file);
    WorkSection.add(m, target1.toText(), ProtobufUtil.toValue(stat1));
    WorkSection.add(m, target2.toText(), ProtobufUtil.toValue(stat2));
    WorkSection.add(m, target3.toText(), ProtobufUtil.toValue(stat3));
    bw.addMutation(m);
    bw.close();
    updater.run();
    try (Scanner s = ReplicationTable.getScanner(conn)) {
        s.setRange(Range.exact(file));
        StatusSection.limit(s);
        Entry<Key, Value> entry = Iterables.getOnlyElement(s);
        Assert.assertEquals(entry.getKey().getColumnFamily(), StatusSection.NAME);
        Assert.assertEquals(entry.getKey().getColumnQualifier().toString(), target1.getSourceTableId().canonicalID());
        // We should only rely on the correct begin attribute being returned
        Status actual = Status.parseFrom(entry.getValue().get());
        Assert.assertEquals(1, actual.getBegin());
    }
}

Example 73 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class GarbageCollectorCommunicatesWithTServersIT method testUnreferencedWalInTserverIsClosed.

@Test(timeout = 2 * 60 * 1000)
public void testUnreferencedWalInTserverIsClosed() throws Exception {
    final String[] names = getUniqueNames(2);
    // `table` will be replicated, `otherTable` is only used to roll the WAL on the tserver
    final String table = names[0], otherTable = names[1];
    final Connector conn = getConnector();
    // Bring the replication table online first and foremost
    ReplicationTable.setOnline(conn);
    log.info("Creating {}", table);
    conn.tableOperations().create(table);
    conn.tableOperations().setProperty(table, Property.TABLE_REPLICATION.getKey(), "true");
    log.info("Writing a few mutations to the table");
    BatchWriter bw = conn.createBatchWriter(table, null);
    byte[] empty = new byte[0];
    for (int i = 0; i < 5; i++) {
        Mutation m = new Mutation(Integer.toString(i));
        m.put(empty, empty, empty);
        bw.addMutation(m);
    }
    log.info("Flushing mutations to the server");
    bw.close();
    log.info("Checking that metadata only has one WAL recorded for this table");
    Set<String> wals = getWalsForTable(table);
    Assert.assertEquals("Expected to only find two WAL for the table", 2, wals.size());
    log.info("Compacting the table which will remove all WALs from the tablets");
    // Flush our test table to remove the WAL references in it
    conn.tableOperations().flush(table, null, null, true);
    // Flush the metadata table too because it will have a reference to the WAL
    conn.tableOperations().flush(MetadataTable.NAME, null, null, true);
    log.info("Fetching replication statuses from metadata table");
    Map<String, Status> fileToStatus = getMetadataStatusForTable(table);
    Assert.assertEquals("Expected to only find one replication status message", 1, fileToStatus.size());
    String walName = fileToStatus.keySet().iterator().next();
    Assert.assertTrue("Expected log file name from tablet to equal replication entry", wals.contains(walName));
    Status status = fileToStatus.get(walName);
    Assert.assertEquals("Expected Status for file to not be closed", false, status.getClosed());
    Set<String> filesForTable = getFilesForTable(table);
    Assert.assertEquals("Expected to only find one rfile for table", 1, filesForTable.size());
    log.info("Files for table before MajC: {}", filesForTable);
    // Issue a MajC to roll a new file in HDFS
    conn.tableOperations().compact(table, null, null, false, true);
    Set<String> filesForTableAfterCompaction = getFilesForTable(table);
    log.info("Files for table after MajC: {}", filesForTableAfterCompaction);
    Assert.assertEquals("Expected to only find one rfile for table", 1, filesForTableAfterCompaction.size());
    Assert.assertNotEquals("Expected the files before and after compaction to differ", filesForTableAfterCompaction, filesForTable);
    // Use the rfile which was just replaced by the MajC to determine when the GC has ran
    Path fileToBeDeleted = new Path(filesForTable.iterator().next());
    FileSystem fs = getCluster().getFileSystem();
    boolean fileExists = fs.exists(fileToBeDeleted);
    while (fileExists) {
        log.info("File which should get deleted still exists: {}", fileToBeDeleted);
        Thread.sleep(2000);
        fileExists = fs.exists(fileToBeDeleted);
    }
    // At this point in time, we *know* that the GarbageCollector has run which means that the Status
    // for our WAL should not be altered.
    Map<String, Status> fileToStatusAfterMinc = getMetadataStatusForTable(table);
    Assert.assertEquals("Expected to still find only one replication status message: " + fileToStatusAfterMinc, 1, fileToStatusAfterMinc.size());
    /*
     * To verify that the WALs is still getting closed, we have to force the tserver to close the existing WAL and open a new one instead. The easiest way to do
     * this is to write a load of data that will exceed the 1.33% full threshold that the logger keeps track of
     */
    conn.tableOperations().create(otherTable);
    bw = conn.createBatchWriter(otherTable, null);
    // 500k
    byte[] bigValue = new byte[1024 * 500];
    Arrays.fill(bigValue, (byte) 1);
    // 500k * 50
    for (int i = 0; i < 50; i++) {
        Mutation m = new Mutation(Integer.toString(i));
        m.put(empty, empty, bigValue);
        bw.addMutation(m);
        if (i % 10 == 0) {
            bw.flush();
        }
    }
    bw.close();
    conn.tableOperations().flush(otherTable, null, null, true);
    // Get the tservers which the master deems as active
    final ClientContext context = new ClientContext(conn.getInstance(), new Credentials("root", new PasswordToken(ConfigurableMacBase.ROOT_PASSWORD)), getClientConfig());
    List<String> tservers = MasterClient.execute(context, new ClientExecReturn<List<String>, MasterClientService.Client>() {

        @Override
        public List<String> execute(MasterClientService.Client client) throws Exception {
            return client.getActiveTservers(Tracer.traceInfo(), context.rpcCreds());
        }
    });
    Assert.assertEquals("Expected only one active tservers", 1, tservers.size());
    HostAndPort tserver = HostAndPort.fromString(tservers.get(0));
    // Get the active WALs from that server
    log.info("Fetching active WALs from {}", tserver);
    Client client = ThriftUtil.getTServerClient(tserver, context);
    List<String> activeWalsForTserver = client.getActiveLogs(Tracer.traceInfo(), context.rpcCreds());
    log.info("Active wals: {}", activeWalsForTserver);
    Assert.assertEquals("Expected to find only one active WAL", 1, activeWalsForTserver.size());
    String activeWal = new Path(activeWalsForTserver.get(0)).toString();
    Assert.assertNotEquals("Current active WAL on tserver should not be the original WAL we saw", walName, activeWal);
    log.info("Ensuring that replication status does get closed after WAL is no longer in use by Tserver");
    do {
        Map<String, Status> replicationStatuses = getMetadataStatusForTable(table);
        log.info("Got replication status messages {}", replicationStatuses);
        Assert.assertEquals("Did not expect to find additional status records", 1, replicationStatuses.size());
        status = replicationStatuses.values().iterator().next();
        log.info("Current status: {}", ProtobufUtil.toString(status));
        if (status.getClosed()) {
            return;
        }
        log.info("Status is not yet closed, waiting for garbage collector to close it");
        Thread.sleep(2000);
    } while (true);
}

Example 74 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class SummaryIT method basicSummaryTest.

@Test
public void basicSummaryTest() throws Exception {
    final String table = getUniqueNames(1)[0];
    Connector c = getConnector();
    NewTableConfiguration ntc = new NewTableConfiguration();
    SummarizerConfiguration sc1 = SummarizerConfiguration.builder(BasicSummarizer.class.getName()).build();
    ntc.enableSummarization(sc1);
    c.tableOperations().create(table, ntc);
    BatchWriter bw = writeData(table, c);
    Collection<Summary> summaries = c.tableOperations().summaries(table).flush(false).retrieve();
    Assert.assertEquals(0, summaries.size());
    LongSummaryStatistics stats = getTimestampStats(table, c);
    summaries = c.tableOperations().summaries(table).flush(true).retrieve();
    checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 100_000l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    Mutation m = new Mutation(String.format("r%09x", 999));
    m.put("f1", "q1", "999-0");
    m.putDelete("f1", "q2");
    bw.addMutation(m);
    bw.flush();
    c.tableOperations().flush(table, null, null, true);
    stats = getTimestampStats(table, c);
    summaries = c.tableOperations().summaries(table).retrieve();
    checkSummaries(summaries, sc1, 2, 0, 0, TOTAL_STAT, 100_002l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 1l);
    bw.close();
    c.tableOperations().compact(table, new CompactionConfig().setWait(true));
    summaries = c.tableOperations().summaries(table).retrieve();
    checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 100_000l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    // split tablet into two
    String sp1 = String.format("r%09x", 50_000);
    addSplits(table, c, sp1);
    summaries = c.tableOperations().summaries(table).retrieve();
    checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 100_000l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    // compact 2nd tablet
    c.tableOperations().compact(table, new CompactionConfig().setStartRow(new Text(sp1)).setWait(true));
    summaries = c.tableOperations().summaries(table).retrieve();
    checkSummaries(summaries, sc1, 2, 0, 1, TOTAL_STAT, 113_999l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    // get summaries for first tablet
    stats = getTimestampStats(table, c, sp1, null);
    summaries = c.tableOperations().summaries(table).startRow(sp1).retrieve();
    checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 49_999l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    // compact all tablets and regenerate all summaries
    c.tableOperations().compact(table, new CompactionConfig());
    summaries = c.tableOperations().summaries(table).retrieve();
    stats = getTimestampStats(table, c);
    checkSummaries(summaries, sc1, 2, 0, 0, TOTAL_STAT, 100_000l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    summaries = c.tableOperations().summaries(table).startRow(String.format("r%09x", 75_000)).endRow(String.format("r%09x", 80_000)).retrieve();
    Summary summary = Iterables.getOnlyElement(summaries);
    Assert.assertEquals(1, summary.getFileStatistics().getTotal());
    Assert.assertEquals(1, summary.getFileStatistics().getExtra());
    long total = summary.getStatistics().get(TOTAL_STAT);
    Assert.assertTrue("Total " + total + " out of expected range", total > 0 && total <= 10_000);
    // test adding and removing
    c.tableOperations().removeSummarizers(table, sc -> sc.getClassName().contains("foo"));
    List<SummarizerConfiguration> summarizers = c.tableOperations().listSummarizers(table);
    Assert.assertEquals(1, summarizers.size());
    Assert.assertTrue(summarizers.contains(sc1));
    c.tableOperations().removeSummarizers(table, sc -> sc.getClassName().equals(BasicSummarizer.class.getName()));
    summarizers = c.tableOperations().listSummarizers(table);
    Assert.assertEquals(0, summarizers.size());
    c.tableOperations().compact(table, new CompactionConfig().setWait(true));
    summaries = c.tableOperations().summaries(table).retrieve();
    Assert.assertEquals(0, summaries.size());
    c.tableOperations().addSummarizers(table, sc1);
    c.tableOperations().compact(table, new CompactionConfig().setWait(true));
    summaries = c.tableOperations().summaries(table).retrieve();
    checkSummaries(summaries, sc1, 2, 0, 0, TOTAL_STAT, 100_000l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
}

Example 75 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class SummaryIT method tooLargeTest.

@Test
public void tooLargeTest() throws Exception {
    final String table = getUniqueNames(1)[0];
    Connector c = getConnector();
    NewTableConfiguration ntc = new NewTableConfiguration();
    SummarizerConfiguration sc1 = SummarizerConfiguration.builder(BigSummarizer.class).build();
    ntc.enableSummarization(sc1);
    c.tableOperations().create(table, ntc);
    try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
        write(bw, "a_large", "f1", "q1", "v1");
        write(bw, "v_small", "f1", "q1", "v2");
    }
    c.tableOperations().flush(table, null, null, true);
    Summary summary = c.tableOperations().summaries(table).retrieve().get(0);
    Assert.assertEquals(1, summary.getFileStatistics().getLarge());
    Assert.assertEquals(0, summary.getFileStatistics().getMissing());
    Assert.assertEquals(0, summary.getFileStatistics().getExtra());
    Assert.assertEquals(0, summary.getFileStatistics().getDeleted());
    Assert.assertEquals(1, summary.getFileStatistics().getInaccurate());
    Assert.assertEquals(1, summary.getFileStatistics().getTotal());
    Assert.assertEquals(Collections.emptyMap(), summary.getStatistics());
    // create situation where one tablet has summary data and one does not because the summary data was too large
    c.tableOperations().addSplits(table, new TreeSet<>(Collections.singleton(new Text("m"))));
    c.tableOperations().compact(table, new CompactionConfig().setWait(true));
    summary = c.tableOperations().summaries(table).retrieve().get(0);
    Assert.assertEquals(1, summary.getFileStatistics().getLarge());
    Assert.assertEquals(0, summary.getFileStatistics().getMissing());
    Assert.assertEquals(0, summary.getFileStatistics().getExtra());
    Assert.assertEquals(0, summary.getFileStatistics().getDeleted());
    Assert.assertEquals(1, summary.getFileStatistics().getInaccurate());
    Assert.assertEquals(2, summary.getFileStatistics().getTotal());
    HashMap<String, Long> expected = new HashMap<>();
    for (int i = 0; i < 10; i++) {
        expected.put(String.format("%09x", i), i * 19l);
    }
    Assert.assertEquals(expected, summary.getStatistics());
}