Examples with BatchWriter org.apache.accumulo.core.client.BatchWriter used on opensource projects

Example 36 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class ReplicationIT method correctRecordsCompleteFile.

@Test
public void correctRecordsCompleteFile() throws Exception {
    Connector conn = getConnector();
    String table = "table1";
    conn.tableOperations().create(table);
    // If we have more than one tserver, this is subject to a race condition.
    conn.tableOperations().setProperty(table, Property.TABLE_REPLICATION.getKey(), "true");
    BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
    for (int i = 0; i < 10; i++) {
        Mutation m = new Mutation(Integer.toString(i));
        m.put(new byte[0], new byte[0], new byte[0]);
        bw.addMutation(m);
    }
    bw.close();
    // After writing data, we'll get a replication table online
    while (!ReplicationTable.isOnline(conn)) {
        sleepUninterruptibly(MILLIS_BETWEEN_REPLICATION_TABLE_ONLINE_CHECKS, TimeUnit.MILLISECONDS);
    }
    Assert.assertTrue("Replication table did not exist", ReplicationTable.isOnline(conn));
    for (int i = 0; i < 5; i++) {
        if (conn.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ)) {
            break;
        }
        log.info("Could not read replication table, waiting and will retry");
        Thread.sleep(2000);
    }
    Assert.assertTrue("'root' user could not read the replication table", conn.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ));
    Set<String> replRows = new HashSet<>();
    int attempts = 5;
    while (replRows.isEmpty() && attempts > 0) {
        try (Scanner scanner = ReplicationTable.getScanner(conn)) {
            StatusSection.limit(scanner);
            for (Entry<Key, Value> entry : scanner) {
                Key k = entry.getKey();
                String fileUri = k.getRow().toString();
                try {
                    new URI(fileUri);
                } catch (URISyntaxException e) {
                    Assert.fail("Expected a valid URI: " + fileUri);
                }
                replRows.add(fileUri);
            }
        }
    }
    Set<String> wals = new HashSet<>();
    attempts = 5;
    Instance i = conn.getInstance();
    ZooReaderWriter zk = new ZooReaderWriter(i.getZooKeepers(), i.getZooKeepersSessionTimeOut(), "");
    while (wals.isEmpty() && attempts > 0) {
        WalStateManager markers = new WalStateManager(i, zk);
        for (Entry<Path, WalState> entry : markers.getAllState().entrySet()) {
            wals.add(entry.getKey().toString());
        }
        attempts--;
    }
    // We only have one file that should need replication (no trace table)
    // We should find an entry in tablet and in the repl row
    Assert.assertEquals("Rows found: " + replRows, 1, replRows.size());
    // There should only be one extra WALog that replication doesn't know about
    replRows.removeAll(wals);
    Assert.assertEquals(2, wals.size());
    Assert.assertEquals(0, replRows.size());
}

Example 37 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class ReplicationIT method filesClosedAfterUnused.

@Test
public void filesClosedAfterUnused() throws Exception {
    Connector conn = getConnector();
    String table = "table";
    conn.tableOperations().create(table);
    Table.ID tableId = Table.ID.of(conn.tableOperations().tableIdMap().get(table));
    Assert.assertNotNull(tableId);
    conn.tableOperations().setProperty(table, Property.TABLE_REPLICATION.getKey(), "true");
    conn.tableOperations().setProperty(table, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
    // just sleep
    conn.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + "cluster1", ReplicaSystemFactory.getPeerConfigurationValue(MockReplicaSystem.class, "50000"));
    // Write a mutation to make a log file
    BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
    Mutation m = new Mutation("one");
    m.put("", "", "");
    bw.addMutation(m);
    bw.close();
    // Write another to make sure the logger rolls itself?
    bw = conn.createBatchWriter(table, new BatchWriterConfig());
    m = new Mutation("three");
    m.put("", "", "");
    bw.addMutation(m);
    bw.close();
    try (Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        s.fetchColumnFamily(TabletsSection.LogColumnFamily.NAME);
        s.setRange(TabletsSection.getRange(tableId));
        Set<String> wals = new HashSet<>();
        for (Entry<Key, Value> entry : s) {
            LogEntry logEntry = LogEntry.fromKeyValue(entry.getKey(), entry.getValue());
            wals.add(new Path(logEntry.filename).toString());
        }
        log.warn("Found wals {}", wals);
        bw = conn.createBatchWriter(table, new BatchWriterConfig());
        m = new Mutation("three");
        byte[] bytes = new byte[1024 * 1024];
        m.put("1".getBytes(), new byte[0], bytes);
        m.put("2".getBytes(), new byte[0], bytes);
        m.put("3".getBytes(), new byte[0], bytes);
        m.put("4".getBytes(), new byte[0], bytes);
        m.put("5".getBytes(), new byte[0], bytes);
        bw.addMutation(m);
        bw.close();
        conn.tableOperations().flush(table, null, null, true);
        while (!ReplicationTable.isOnline(conn)) {
            sleepUninterruptibly(MILLIS_BETWEEN_REPLICATION_TABLE_ONLINE_CHECKS, TimeUnit.MILLISECONDS);
        }
        for (int i = 0; i < 10; i++) {
            try (Scanner s2 = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
                s2.fetchColumnFamily(LogColumnFamily.NAME);
                s2.setRange(TabletsSection.getRange(tableId));
                for (Entry<Key, Value> entry : s2) {
                    log.info("{}={}", entry.getKey().toStringNoTruncate(), entry.getValue());
                }
            }
            try (Scanner s3 = ReplicationTable.getScanner(conn)) {
                StatusSection.limit(s3);
                Text buff = new Text();
                boolean allReferencedLogsClosed = true;
                int recordsFound = 0;
                for (Entry<Key, Value> e : s3) {
                    recordsFound++;
                    allReferencedLogsClosed = true;
                    StatusSection.getFile(e.getKey(), buff);
                    String file = buff.toString();
                    if (wals.contains(file)) {
                        Status stat = Status.parseFrom(e.getValue().get());
                        if (!stat.getClosed()) {
                            log.info("{} wasn't closed", file);
                            allReferencedLogsClosed = false;
                        }
                    }
                }
                if (recordsFound > 0 && allReferencedLogsClosed) {
                    return;
                }
                Thread.sleep(2000);
            } catch (RuntimeException e) {
                Throwable cause = e.getCause();
                if (cause instanceof AccumuloSecurityException) {
                    AccumuloSecurityException ase = (AccumuloSecurityException) cause;
                    switch(ase.getSecurityErrorCode()) {
                        case PERMISSION_DENIED:
                            // We tried to read the replication table before the GRANT went through
                            Thread.sleep(2000);
                            break;
                        default:
                            throw e;
                    }
                }
            }
        }
        Assert.fail("We had a file that was referenced but didn't get closed");
    }
}

Example 38 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class ReplicationOperationsImplIT method waitsUntilEntriesAreReplicated.

@Test
public void waitsUntilEntriesAreReplicated() throws Exception {
    conn.tableOperations().create("foo");
    Table.ID tableId = Table.ID.of(conn.tableOperations().tableIdMap().get("foo"));
    String file1 = "/accumulo/wals/tserver+port/" + UUID.randomUUID(), file2 = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    Status stat = Status.newBuilder().setBegin(0).setEnd(10000).setInfiniteEnd(false).setClosed(false).build();
    BatchWriter bw = ReplicationTable.getBatchWriter(conn);
    Mutation m = new Mutation(file1);
    StatusSection.add(m, tableId, ProtobufUtil.toValue(stat));
    bw.addMutation(m);
    m = new Mutation(file2);
    StatusSection.add(m, tableId, ProtobufUtil.toValue(stat));
    bw.addMutation(m);
    bw.close();
    bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
    m = new Mutation(ReplicationSection.getRowPrefix() + file1);
    m.put(ReplicationSection.COLF, new Text(tableId.getUtf8()), ProtobufUtil.toValue(stat));
    bw.addMutation(m);
    m = new Mutation(ReplicationSection.getRowPrefix() + file2);
    m.put(ReplicationSection.COLF, new Text(tableId.getUtf8()), ProtobufUtil.toValue(stat));
    bw.close();
    final AtomicBoolean done = new AtomicBoolean(false);
    final AtomicBoolean exception = new AtomicBoolean(false);
    final ReplicationOperationsImpl roi = getReplicationOperations();
    Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                roi.drain("foo");
            } catch (Exception e) {
                log.error("Got error", e);
                exception.set(true);
            }
            done.set(true);
        }
    });
    t.start();
    // With the records, we shouldn't be drained
    Assert.assertFalse(done.get());
    bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
    m = new Mutation(ReplicationSection.getRowPrefix() + file1);
    m.putDelete(ReplicationSection.COLF, new Text(tableId.getUtf8()));
    bw.addMutation(m);
    bw.flush();
    Assert.assertFalse(done.get());
    m = new Mutation(ReplicationSection.getRowPrefix() + file2);
    m.putDelete(ReplicationSection.COLF, new Text(tableId.getUtf8()));
    bw.addMutation(m);
    bw.flush();
    bw.close();
    // Removing metadata entries doesn't change anything
    Assert.assertFalse(done.get());
    // Remove the replication entries too
    bw = ReplicationTable.getBatchWriter(conn);
    m = new Mutation(file1);
    m.putDelete(StatusSection.NAME, new Text(tableId.getUtf8()));
    bw.addMutation(m);
    bw.flush();
    Assert.assertFalse(done.get());
    m = new Mutation(file2);
    m.putDelete(StatusSection.NAME, new Text(tableId.getUtf8()));
    bw.addMutation(m);
    bw.flush();
    try {
        t.join(5000);
    } catch (InterruptedException e) {
        Assert.fail("ReplicationOperations.drain did not complete");
    }
    // After both metadata and replication
    Assert.assertTrue("Drain never finished", done.get());
    Assert.assertFalse("Saw unexpectetd exception", exception.get());
}

Example 39 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class SequentialWorkAssignerIT method workAcrossTablesHappensConcurrently.

@Test
public void workAcrossTablesHappensConcurrently() throws Exception {
    ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1", Table.ID.of("1"));
    Text serializedTarget1 = target1.toText();
    ReplicationTarget target2 = new ReplicationTarget("cluster1", "table2", Table.ID.of("2"));
    Text serializedTarget2 = target2.toText();
    // Create two mutations, both of which need replication work done
    BatchWriter bw = ReplicationTable.getBatchWriter(conn);
    // We want the name of file2 to sort before file1
    String filename1 = "z_file1", filename2 = "a_file1";
    String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2;
    // File1 was closed before file2, however
    Status stat1 = Status.newBuilder().setBegin(0).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(250).build();
    Status stat2 = Status.newBuilder().setBegin(0).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(500).build();
    Mutation m = new Mutation(file1);
    WorkSection.add(m, serializedTarget1, ProtobufUtil.toValue(stat1));
    bw.addMutation(m);
    m = new Mutation(file2);
    WorkSection.add(m, serializedTarget2, ProtobufUtil.toValue(stat2));
    bw.addMutation(m);
    m = OrderSection.createMutation(file1, stat1.getCreatedTime());
    OrderSection.add(m, target1.getSourceTableId(), ProtobufUtil.toValue(stat1));
    bw.addMutation(m);
    m = OrderSection.createMutation(file2, stat2.getCreatedTime());
    OrderSection.add(m, target2.getSourceTableId(), ProtobufUtil.toValue(stat2));
    bw.addMutation(m);
    bw.close();
    DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class);
    Map<String, Map<Table.ID, String>> queuedWork = new HashMap<>();
    assigner.setQueuedWork(queuedWork);
    assigner.setWorkQueue(workQueue);
    assigner.setMaxQueueSize(Integer.MAX_VALUE);
    // Make sure we expect the invocations in the correct order (accumulo is sorted)
    workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target1), file1);
    expectLastCall().once();
    workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target2), file2);
    expectLastCall().once();
    // file2 is *not* queued because file1 must be replicated first
    replay(workQueue);
    assigner.createWork();
    verify(workQueue);
    Assert.assertEquals(1, queuedWork.size());
    Assert.assertTrue(queuedWork.containsKey("cluster1"));
    Map<Table.ID, String> cluster1Work = queuedWork.get("cluster1");
    Assert.assertEquals(2, cluster1Work.size());
    Assert.assertTrue(cluster1Work.containsKey(target1.getSourceTableId()));
    Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target1), cluster1Work.get(target1.getSourceTableId()));
    Assert.assertTrue(cluster1Work.containsKey(target2.getSourceTableId()));
    Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target2), cluster1Work.get(target2.getSourceTableId()));
}

Example 40 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class SequentialWorkAssignerIT method createWorkForFilesInCorrectOrder.

@Test
public void createWorkForFilesInCorrectOrder() throws Exception {
    ReplicationTarget target = new ReplicationTarget("cluster1", "table1", Table.ID.of("1"));
    Text serializedTarget = target.toText();
    // Create two mutations, both of which need replication work done
    BatchWriter bw = ReplicationTable.getBatchWriter(conn);
    // We want the name of file2 to sort before file1
    String filename1 = "z_file1", filename2 = "a_file1";
    String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2;
    // File1 was closed before file2, however
    Status stat1 = Status.newBuilder().setBegin(0).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(250).build();
    Status stat2 = Status.newBuilder().setBegin(0).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(500).build();
    Mutation m = new Mutation(file1);
    WorkSection.add(m, serializedTarget, ProtobufUtil.toValue(stat1));
    bw.addMutation(m);
    m = new Mutation(file2);
    WorkSection.add(m, serializedTarget, ProtobufUtil.toValue(stat2));
    bw.addMutation(m);
    m = OrderSection.createMutation(file1, stat1.getCreatedTime());
    OrderSection.add(m, target.getSourceTableId(), ProtobufUtil.toValue(stat1));
    bw.addMutation(m);
    m = OrderSection.createMutation(file2, stat2.getCreatedTime());
    OrderSection.add(m, target.getSourceTableId(), ProtobufUtil.toValue(stat2));
    bw.addMutation(m);
    bw.close();
    DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class);
    Map<String, Map<Table.ID, String>> queuedWork = new HashMap<>();
    assigner.setQueuedWork(queuedWork);
    assigner.setWorkQueue(workQueue);
    assigner.setMaxQueueSize(Integer.MAX_VALUE);
    // Make sure we expect the invocations in the correct order (accumulo is sorted)
    workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target), file1);
    expectLastCall().once();
    // file2 is *not* queued because file1 must be replicated first
    replay(workQueue);
    assigner.createWork();
    verify(workQueue);
    Assert.assertEquals(1, queuedWork.size());
    Assert.assertTrue(queuedWork.containsKey("cluster1"));
    Map<Table.ID, String> cluster1Work = queuedWork.get("cluster1");
    Assert.assertEquals(1, cluster1Work.size());
    Assert.assertTrue(cluster1Work.containsKey(target.getSourceTableId()));
    Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target), cluster1Work.get(target.getSourceTableId()));
}