Examples with BatchWriterConfig org.apache.accumulo.core.client.BatchWriterConfig used on opensource projects

Example 31 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class BatchWriterInTabletServerIT method test.

private void test(String t1, String t2, Connector c, IteratorSetting itset, int numEntriesToWritePerEntry) throws Exception {
    // Write an entry to t1
    c.tableOperations().create(t1);
    Key k = new Key(new Text("row"), new Text("cf"), new Text("cq"));
    Value v = new Value("1".getBytes());
    {
        BatchWriterConfig config = new BatchWriterConfig();
        config.setMaxMemory(0);
        BatchWriter writer = c.createBatchWriter(t1, config);
        Mutation m = new Mutation(k.getRow());
        m.put(k.getColumnFamily(), k.getColumnQualifier(), v);
        writer.addMutation(m);
        writer.close();
    }
    // Create t2 with a combiner to count entries written to it
    c.tableOperations().create(t2);
    IteratorSetting summer = new IteratorSetting(2, "summer", SummingCombiner.class);
    LongCombiner.setEncodingType(summer, LongCombiner.Type.STRING);
    LongCombiner.setCombineAllColumns(summer, true);
    c.tableOperations().attachIterator(t2, summer);
    Map.Entry<Key, Value> actual;
    try (Scanner scanner = c.createScanner(t1, Authorizations.EMPTY)) {
        // Scan t1 with an iterator that writes to table t2
        scanner.addScanIterator(itset);
        actual = Iterators.getOnlyElement(scanner.iterator());
        Assert.assertTrue(actual.getKey().equals(k, PartialKey.ROW_COLFAM_COLQUAL));
        Assert.assertEquals(BatchWriterIterator.SUCCESS_VALUE, actual.getValue());
    }
    try (Scanner scanner = c.createScanner(t2, Authorizations.EMPTY)) {
        // ensure entries correctly wrote to table t2
        actual = Iterators.getOnlyElement(scanner.iterator());
        log.debug("t2 entry is " + actual.getKey().toStringNoTime() + " -> " + actual.getValue());
        Assert.assertTrue(actual.getKey().equals(k, PartialKey.ROW_COLFAM_COLQUAL));
        Assert.assertEquals(numEntriesToWritePerEntry, Integer.parseInt(actual.getValue().toString()));
    }
    c.tableOperations().delete(t1);
    c.tableOperations().delete(t2);
}

Example 32 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class BatchWriterFlushIT method runLatencyTest.

private void runLatencyTest(String tableName) throws Exception {
    // should automatically flush after 2 seconds
    try (BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig().setMaxLatency(1000, TimeUnit.MILLISECONDS));
        Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY)) {
        Mutation m = new Mutation(new Text(String.format("r_%10d", 1)));
        m.put(new Text("cf"), new Text("cq"), new Value("1".getBytes(UTF_8)));
        bw.addMutation(m);
        sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
        int count = Iterators.size(scanner.iterator());
        if (count != 0) {
            throw new Exception("Flushed too soon");
        }
        sleepUninterruptibly(1500, TimeUnit.MILLISECONDS);
        count = Iterators.size(scanner.iterator());
        if (count != 1) {
            throw new Exception("Did not flush");
        }
    }
}

Example 33 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class MultiInstanceReplicationIT method dataWasReplicatedToThePeer.

@Test(timeout = 10 * 60 * 1000)
public void dataWasReplicatedToThePeer() throws Exception {
    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
    peerCfg.setNumTservers(1);
    peerCfg.setInstanceName("peer");
    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
    MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
    peerCluster.start();
    try {
        final Connector connMaster = getConnector();
        final Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
        ReplicationTable.setOnline(connMaster);
        String peerUserName = "peer", peerPassword = "foo";
        String peerClusterName = "peer";
        connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
        // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
        final String masterTable = "master", peerTable = "peer";
        connMaster.tableOperations().create(masterTable);
        String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
        Assert.assertNotNull(masterTableId);
        connPeer.tableOperations().create(peerTable);
        String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
        Assert.assertNotNull(peerTableId);
        connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
        // Replicate this table to the peerClusterName in a table with the peerTableId table id
        connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
        connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
        // Write some data to table1
        BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
        for (int rows = 0; rows < 5000; rows++) {
            Mutation m = new Mutation(Integer.toString(rows));
            for (int cols = 0; cols < 100; cols++) {
                String value = Integer.toString(cols);
                m.put(value, "", value);
            }
            bw.addMutation(m);
        }
        bw.close();
        log.info("Wrote all data to master cluster");
        final Set<String> filesNeedingReplication = connMaster.replicationOperations().referencedFiles(masterTable);
        log.info("Files to replicate: " + filesNeedingReplication);
        for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        cluster.exec(TabletServer.class);
        log.info("TabletServer restarted");
        Iterators.size(ReplicationTable.getScanner(connMaster).iterator());
        log.info("TabletServer is online");
        while (!ReplicationTable.isOnline(connMaster)) {
            log.info("Replication table still offline, waiting");
            Thread.sleep(5000);
        }
        log.info("");
        log.info("Fetching metadata records:");
        for (Entry<Key, Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
            } else {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), kv.getValue());
            }
        }
        log.info("");
        log.info("Fetching replication records:");
        for (Entry<Key, Value> kv : ReplicationTable.getScanner(connMaster)) {
            log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
        }
        Future<Boolean> future = executor.submit(new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                long then = System.currentTimeMillis();
                connMaster.replicationOperations().drain(masterTable, filesNeedingReplication);
                long now = System.currentTimeMillis();
                log.info("Drain completed in " + (now - then) + "ms");
                return true;
            }
        });
        try {
            future.get(60, TimeUnit.SECONDS);
        } catch (TimeoutException e) {
            future.cancel(true);
            Assert.fail("Drain did not finish within 60 seconds");
        } finally {
            executor.shutdownNow();
        }
        log.info("drain completed");
        log.info("");
        log.info("Fetching metadata records:");
        for (Entry<Key, Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
            } else {
                log.info("{} {}", kv.getKey().toStringNoTruncate(), kv.getValue());
            }
        }
        log.info("");
        log.info("Fetching replication records:");
        for (Entry<Key, Value> kv : ReplicationTable.getScanner(connMaster)) {
            log.info("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
        }
        try (Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY);
            Scanner peer = connPeer.createScanner(peerTable, Authorizations.EMPTY)) {
            Iterator<Entry<Key, Value>> masterIter = master.iterator(), peerIter = peer.iterator();
            Entry<Key, Value> masterEntry = null, peerEntry = null;
            while (masterIter.hasNext() && peerIter.hasNext()) {
                masterEntry = masterIter.next();
                peerEntry = peerIter.next();
                Assert.assertEquals(masterEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0, masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
                Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
            }
            log.info("Last master entry: {}", masterEntry);
            log.info("Last peer entry: {}", peerEntry);
            Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
            Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
        }
    } finally {
        peerCluster.stop();
    }
}

Example 34 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class MultiInstanceReplicationIT method dataReplicatedToCorrectTableWithoutDrain.

@Test
public void dataReplicatedToCorrectTableWithoutDrain() throws Exception {
    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
    peerCfg.setNumTservers(1);
    peerCfg.setInstanceName("peer");
    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
    MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
    peer1Cluster.start();
    try {
        Connector connMaster = getConnector();
        Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
        String peerClusterName = "peer";
        String peerUserName = "repl";
        String peerPassword = "passwd";
        // Create a user on the peer for replication to use
        connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
        // Configure the credentials we should use to authenticate ourselves to the peer for replication
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
        // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
        String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
        connMaster.tableOperations().create(masterTable1);
        String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
        Assert.assertNotNull(masterTableId1);
        connMaster.tableOperations().create(masterTable2);
        String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
        Assert.assertNotNull(masterTableId2);
        connPeer.tableOperations().create(peerTable1);
        String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
        Assert.assertNotNull(peerTableId1);
        connPeer.tableOperations().create(peerTable2);
        String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
        Assert.assertNotNull(peerTableId2);
        // Give our replication user the ability to write to the tables
        connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
        connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
        // Replicate this table to the peerClusterName in a table with the peerTableId table id
        connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
        connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
        connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
        connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
        // Write some data to table1
        BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
        for (int rows = 0; rows < 2500; rows++) {
            Mutation m = new Mutation(masterTable1 + rows);
            for (int cols = 0; cols < 100; cols++) {
                String value = Integer.toString(cols);
                m.put(value, "", value);
            }
            bw.addMutation(m);
        }
        bw.close();
        // Write some data to table2
        bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
        for (int rows = 0; rows < 2500; rows++) {
            Mutation m = new Mutation(masterTable2 + rows);
            for (int cols = 0; cols < 100; cols++) {
                String value = Integer.toString(cols);
                m.put(value, "", value);
            }
            bw.addMutation(m);
        }
        bw.close();
        log.info("Wrote all data to master cluster");
        for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        cluster.exec(TabletServer.class);
        while (!ReplicationTable.isOnline(connMaster)) {
            log.info("Replication table still offline, waiting");
            Thread.sleep(5000);
        }
        // Wait until we fully replicated something
        boolean fullyReplicated = false;
        for (int i = 0; i < 10 && !fullyReplicated; i++) {
            sleepUninterruptibly(2, TimeUnit.SECONDS);
            try (Scanner s = ReplicationTable.getScanner(connMaster)) {
                WorkSection.limit(s);
                for (Entry<Key, Value> entry : s) {
                    Status status = Status.parseFrom(entry.getValue().get());
                    if (StatusUtil.isFullyReplicated(status)) {
                        fullyReplicated |= true;
                    }
                }
            }
        }
        Assert.assertNotEquals(0, fullyReplicated);
        // We have to wait for the master to assign the replication work, a local tserver to process it, and then the remote tserver to replay it
        // Be cautious in how quickly we assert that the data is present on the peer
        long countTable = 0l;
        for (int i = 0; i < 10; i++) {
            for (Entry<Key, Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
                countTable++;
                Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(masterTable1));
            }
            log.info("Found {} records in {}", countTable, peerTable1);
            if (0l == countTable) {
                Thread.sleep(5000);
            } else {
                break;
            }
        }
        Assert.assertTrue("Found no records in " + peerTable1 + " in the peer cluster", countTable > 0);
        // Be cautious in how quickly we assert that the data is present on the peer
        for (int i = 0; i < 10; i++) {
            countTable = 0l;
            for (Entry<Key, Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
                countTable++;
                Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(masterTable2));
            }
            log.info("Found {} records in {}", countTable, peerTable2);
            if (0l == countTable) {
                Thread.sleep(5000);
            } else {
                break;
            }
        }
        Assert.assertTrue("Found no records in " + peerTable2 + " in the peer cluster", countTable > 0);
    } finally {
        peer1Cluster.stop();
    }
}

Example 35 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class ReplicationIT method correctRecordsCompleteFile.

@Test
public void correctRecordsCompleteFile() throws Exception {
    Connector conn = getConnector();
    String table = "table1";
    conn.tableOperations().create(table);
    // If we have more than one tserver, this is subject to a race condition.
    conn.tableOperations().setProperty(table, Property.TABLE_REPLICATION.getKey(), "true");
    BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
    for (int i = 0; i < 10; i++) {
        Mutation m = new Mutation(Integer.toString(i));
        m.put(new byte[0], new byte[0], new byte[0]);
        bw.addMutation(m);
    }
    bw.close();
    // After writing data, we'll get a replication table online
    while (!ReplicationTable.isOnline(conn)) {
        sleepUninterruptibly(MILLIS_BETWEEN_REPLICATION_TABLE_ONLINE_CHECKS, TimeUnit.MILLISECONDS);
    }
    Assert.assertTrue("Replication table did not exist", ReplicationTable.isOnline(conn));
    for (int i = 0; i < 5; i++) {
        if (conn.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ)) {
            break;
        }
        log.info("Could not read replication table, waiting and will retry");
        Thread.sleep(2000);
    }
    Assert.assertTrue("'root' user could not read the replication table", conn.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ));
    Set<String> replRows = new HashSet<>();
    int attempts = 5;
    while (replRows.isEmpty() && attempts > 0) {
        try (Scanner scanner = ReplicationTable.getScanner(conn)) {
            StatusSection.limit(scanner);
            for (Entry<Key, Value> entry : scanner) {
                Key k = entry.getKey();
                String fileUri = k.getRow().toString();
                try {
                    new URI(fileUri);
                } catch (URISyntaxException e) {
                    Assert.fail("Expected a valid URI: " + fileUri);
                }
                replRows.add(fileUri);
            }
        }
    }
    Set<String> wals = new HashSet<>();
    attempts = 5;
    Instance i = conn.getInstance();
    ZooReaderWriter zk = new ZooReaderWriter(i.getZooKeepers(), i.getZooKeepersSessionTimeOut(), "");
    while (wals.isEmpty() && attempts > 0) {
        WalStateManager markers = new WalStateManager(i, zk);
        for (Entry<Path, WalState> entry : markers.getAllState().entrySet()) {
            wals.add(entry.getKey().toString());
        }
        attempts--;
    }
    // We only have one file that should need replication (no trace table)
    // We should find an entry in tablet and in the repl row
    Assert.assertEquals("Rows found: " + replRows, 1, replRows.size());
    // There should only be one extra WALog that replication doesn't know about
    replRows.removeAll(wals);
    Assert.assertEquals(2, wals.size());
    Assert.assertEquals(0, replRows.size());
}