Examples with Status org.apache.accumulo.server.replication.proto.Replication.Status used on opensource projects

Example 6 with Status

use of org.apache.accumulo.server.replication.proto.Replication.Status in project accumulo by apache.

the class SequentialWorkAssignerIT method workAcrossTablesHappensConcurrently.

@Test
public void workAcrossTablesHappensConcurrently() throws Exception {
    ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1", Table.ID.of("1"));
    Text serializedTarget1 = target1.toText();
    ReplicationTarget target2 = new ReplicationTarget("cluster1", "table2", Table.ID.of("2"));
    Text serializedTarget2 = target2.toText();
    // Create two mutations, both of which need replication work done
    BatchWriter bw = ReplicationTable.getBatchWriter(conn);
    // We want the name of file2 to sort before file1
    String filename1 = "z_file1", filename2 = "a_file1";
    String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2;
    // File1 was closed before file2, however
    Status stat1 = Status.newBuilder().setBegin(0).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(250).build();
    Status stat2 = Status.newBuilder().setBegin(0).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(500).build();
    Mutation m = new Mutation(file1);
    WorkSection.add(m, serializedTarget1, ProtobufUtil.toValue(stat1));
    bw.addMutation(m);
    m = new Mutation(file2);
    WorkSection.add(m, serializedTarget2, ProtobufUtil.toValue(stat2));
    bw.addMutation(m);
    m = OrderSection.createMutation(file1, stat1.getCreatedTime());
    OrderSection.add(m, target1.getSourceTableId(), ProtobufUtil.toValue(stat1));
    bw.addMutation(m);
    m = OrderSection.createMutation(file2, stat2.getCreatedTime());
    OrderSection.add(m, target2.getSourceTableId(), ProtobufUtil.toValue(stat2));
    bw.addMutation(m);
    bw.close();
    DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class);
    Map<String, Map<Table.ID, String>> queuedWork = new HashMap<>();
    assigner.setQueuedWork(queuedWork);
    assigner.setWorkQueue(workQueue);
    assigner.setMaxQueueSize(Integer.MAX_VALUE);
    // Make sure we expect the invocations in the correct order (accumulo is sorted)
    workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target1), file1);
    expectLastCall().once();
    workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target2), file2);
    expectLastCall().once();
    // file2 is *not* queued because file1 must be replicated first
    replay(workQueue);
    assigner.createWork();
    verify(workQueue);
    Assert.assertEquals(1, queuedWork.size());
    Assert.assertTrue(queuedWork.containsKey("cluster1"));
    Map<Table.ID, String> cluster1Work = queuedWork.get("cluster1");
    Assert.assertEquals(2, cluster1Work.size());
    Assert.assertTrue(cluster1Work.containsKey(target1.getSourceTableId()));
    Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target1), cluster1Work.get(target1.getSourceTableId()));
    Assert.assertTrue(cluster1Work.containsKey(target2.getSourceTableId()));
    Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target2), cluster1Work.get(target2.getSourceTableId()));
}

Example 7 with Status

use of org.apache.accumulo.server.replication.proto.Replication.Status in project accumulo by apache.

the class SequentialWorkAssignerIT method createWorkForFilesInCorrectOrder.

@Test
public void createWorkForFilesInCorrectOrder() throws Exception {
    ReplicationTarget target = new ReplicationTarget("cluster1", "table1", Table.ID.of("1"));
    Text serializedTarget = target.toText();
    // Create two mutations, both of which need replication work done
    BatchWriter bw = ReplicationTable.getBatchWriter(conn);
    // We want the name of file2 to sort before file1
    String filename1 = "z_file1", filename2 = "a_file1";
    String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2;
    // File1 was closed before file2, however
    Status stat1 = Status.newBuilder().setBegin(0).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(250).build();
    Status stat2 = Status.newBuilder().setBegin(0).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(500).build();
    Mutation m = new Mutation(file1);
    WorkSection.add(m, serializedTarget, ProtobufUtil.toValue(stat1));
    bw.addMutation(m);
    m = new Mutation(file2);
    WorkSection.add(m, serializedTarget, ProtobufUtil.toValue(stat2));
    bw.addMutation(m);
    m = OrderSection.createMutation(file1, stat1.getCreatedTime());
    OrderSection.add(m, target.getSourceTableId(), ProtobufUtil.toValue(stat1));
    bw.addMutation(m);
    m = OrderSection.createMutation(file2, stat2.getCreatedTime());
    OrderSection.add(m, target.getSourceTableId(), ProtobufUtil.toValue(stat2));
    bw.addMutation(m);
    bw.close();
    DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class);
    Map<String, Map<Table.ID, String>> queuedWork = new HashMap<>();
    assigner.setQueuedWork(queuedWork);
    assigner.setWorkQueue(workQueue);
    assigner.setMaxQueueSize(Integer.MAX_VALUE);
    // Make sure we expect the invocations in the correct order (accumulo is sorted)
    workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target), file1);
    expectLastCall().once();
    // file2 is *not* queued because file1 must be replicated first
    replay(workQueue);
    assigner.createWork();
    verify(workQueue);
    Assert.assertEquals(1, queuedWork.size());
    Assert.assertTrue(queuedWork.containsKey("cluster1"));
    Map<Table.ID, String> cluster1Work = queuedWork.get("cluster1");
    Assert.assertEquals(1, cluster1Work.size());
    Assert.assertTrue(cluster1Work.containsKey(target.getSourceTableId()));
    Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target), cluster1Work.get(target.getSourceTableId()));
}

Example 8 with Status

use of org.apache.accumulo.server.replication.proto.Replication.Status in project accumulo by apache.

the class StatusMakerIT method orderRecordsCreatedWithNoCreatedTime.

@Test
public void orderRecordsCreatedWithNoCreatedTime() throws Exception {
    String sourceTable = testName.getMethodName();
    conn.tableOperations().create(sourceTable);
    ReplicationTableUtil.configureMetadataTable(conn, sourceTable);
    BatchWriter bw = conn.createBatchWriter(sourceTable, new BatchWriterConfig());
    String walPrefix = "hdfs://localhost:8020/accumulo/wals/tserver+port/";
    List<String> files = Arrays.asList(walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID());
    Map<String, Long> fileToTableId = new HashMap<>();
    Status.Builder statBuilder = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(true);
    Map<String, Long> statuses = new HashMap<>();
    long index = 1;
    for (String file : files) {
        Mutation m = new Mutation(ReplicationSection.getRowPrefix() + file);
        m.put(ReplicationSection.COLF, new Text(Long.toString(index)), ProtobufUtil.toValue(statBuilder.build()));
        bw.addMutation(m);
        fileToTableId.put(file, index);
        FileStatus status = EasyMock.mock(FileStatus.class);
        EasyMock.expect(status.getModificationTime()).andReturn(index);
        EasyMock.replay(status);
        statuses.put(file, index);
        EasyMock.expect(fs.exists(new Path(file))).andReturn(true);
        EasyMock.expect(fs.getFileStatus(new Path(file))).andReturn(status);
        index++;
    }
    EasyMock.replay(fs);
    bw.close();
    StatusMaker statusMaker = new StatusMaker(conn, fs);
    statusMaker.setSourceTableName(sourceTable);
    statusMaker.run();
    Scanner s = conn.createScanner(sourceTable, Authorizations.EMPTY);
    s.setRange(ReplicationSection.getRange());
    s.fetchColumnFamily(ReplicationSection.COLF);
    Assert.assertEquals(0, Iterables.size(s));
    s = ReplicationTable.getScanner(conn);
    OrderSection.limit(s);
    Iterator<Entry<Key, Value>> iter = s.iterator();
    Assert.assertTrue("Found no order records in replication table", iter.hasNext());
    Iterator<String> expectedFiles = files.iterator();
    Text buff = new Text();
    while (expectedFiles.hasNext() && iter.hasNext()) {
        String file = expectedFiles.next();
        Entry<Key, Value> entry = iter.next();
        Assert.assertEquals(file, OrderSection.getFile(entry.getKey(), buff));
        OrderSection.getTableId(entry.getKey(), buff);
        Assert.assertEquals(fileToTableId.get(file).intValue(), Integer.parseInt(buff.toString()));
        Status status = Status.parseFrom(entry.getValue().get());
        Assert.assertTrue(status.hasCreatedTime());
        Assert.assertEquals((long) statuses.get(file), status.getCreatedTime());
    }
    Assert.assertFalse("Found more files unexpectedly", expectedFiles.hasNext());
    Assert.assertFalse("Found more entries in replication table unexpectedly", iter.hasNext());
    s = conn.createScanner(sourceTable, Authorizations.EMPTY);
    s.setRange(ReplicationSection.getRange());
    s.fetchColumnFamily(ReplicationSection.COLF);
    Assert.assertEquals(0, Iterables.size(s));
    s = ReplicationTable.getScanner(conn);
    s.setRange(ReplicationSection.getRange());
    iter = s.iterator();
    Assert.assertTrue("Found no stat records in replication table", iter.hasNext());
    Collections.sort(files);
    expectedFiles = files.iterator();
    while (expectedFiles.hasNext() && iter.hasNext()) {
        String file = expectedFiles.next();
        Entry<Key, Value> entry = iter.next();
        Status status = Status.parseFrom(entry.getValue().get());
        Assert.assertTrue(status.hasCreatedTime());
        Assert.assertEquals((long) statuses.get(file), status.getCreatedTime());
    }
    Assert.assertFalse("Found more files unexpectedly", expectedFiles.hasNext());
    Assert.assertFalse("Found more entries in replication table unexpectedly", iter.hasNext());
}

Example 9 with Status

use of org.apache.accumulo.server.replication.proto.Replication.Status in project accumulo by apache.

the class UnorderedWorkAssignerReplicationIT method dataReplicatedToCorrectTableWithoutDrain.

@Test
public void dataReplicatedToCorrectTableWithoutDrain() throws Exception {
    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
    peerCfg.setNumTservers(1);
    peerCfg.setInstanceName("peer");
    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
    MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
    peer1Cluster.start();
    try {
        Connector connMaster = getConnector();
        Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
        String peerClusterName = "peer";
        String peerUserName = "repl";
        String peerPassword = "passwd";
        // Create a user on the peer for replication to use
        connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
        // Configure the credentials we should use to authenticate ourselves to the peer for replication
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
        // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
        connMaster.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
        String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
        connMaster.tableOperations().create(masterTable1);
        String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
        Assert.assertNotNull(masterTableId1);
        connMaster.tableOperations().create(masterTable2);
        String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
        Assert.assertNotNull(masterTableId2);
        connPeer.tableOperations().create(peerTable1);
        String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
        Assert.assertNotNull(peerTableId1);
        connPeer.tableOperations().create(peerTable2);
        String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
        Assert.assertNotNull(peerTableId2);
        // Give our replication user the ability to write to the tables
        connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
        connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
        // Replicate this table to the peerClusterName in a table with the peerTableId table id
        connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
        connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
        connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
        connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
        // Wait for zookeeper updates (configuration) to propagate
        sleepUninterruptibly(3, TimeUnit.SECONDS);
        // Write some data to table1
        BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
        for (int rows = 0; rows < 2500; rows++) {
            Mutation m = new Mutation(masterTable1 + rows);
            for (int cols = 0; cols < 100; cols++) {
                String value = Integer.toString(cols);
                m.put(value, "", value);
            }
            bw.addMutation(m);
        }
        bw.close();
        // Write some data to table2
        bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
        for (int rows = 0; rows < 2500; rows++) {
            Mutation m = new Mutation(masterTable2 + rows);
            for (int cols = 0; cols < 100; cols++) {
                String value = Integer.toString(cols);
                m.put(value, "", value);
            }
            bw.addMutation(m);
        }
        bw.close();
        log.info("Wrote all data to master cluster");
        while (!ReplicationTable.isOnline(connMaster)) {
            Thread.sleep(500);
        }
        for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
            cluster.killProcess(ServerType.TABLET_SERVER, proc);
        }
        cluster.exec(TabletServer.class);
        // Wait until we fully replicated something
        boolean fullyReplicated = false;
        for (int i = 0; i < 10 && !fullyReplicated; i++) {
            sleepUninterruptibly(timeoutFactor * 2, TimeUnit.SECONDS);
            try (Scanner s = ReplicationTable.getScanner(connMaster)) {
                WorkSection.limit(s);
                for (Entry<Key, Value> entry : s) {
                    Status status = Status.parseFrom(entry.getValue().get());
                    if (StatusUtil.isFullyReplicated(status)) {
                        fullyReplicated |= true;
                    }
                }
            }
        }
        Assert.assertNotEquals(0, fullyReplicated);
        long countTable = 0l;
        // Check a few times
        for (int i = 0; i < 10; i++) {
            countTable = 0l;
            for (Entry<Key, Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
                countTable++;
                Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(masterTable1));
            }
            log.info("Found {} records in {}", countTable, peerTable1);
            if (0 < countTable) {
                break;
            }
            Thread.sleep(2000);
        }
        Assert.assertTrue("Did not find any records in " + peerTable1 + " on peer", countTable > 0);
        for (int i = 0; i < 10; i++) {
            countTable = 0l;
            for (Entry<Key, Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
                countTable++;
                Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(masterTable2));
            }
            log.info("Found {} records in {}", countTable, peerTable2);
            if (0 < countTable) {
                break;
            }
            Thread.sleep(2000);
        }
        Assert.assertTrue("Did not find any records in " + peerTable2 + " on peer", countTable > 0);
    } finally {
        peer1Cluster.stop();
    }
}

Example 10 with Status

use of org.apache.accumulo.server.replication.proto.Replication.Status in project accumulo by apache.

the class CloseWriteAheadLogReferences method updateReplicationEntries.

/**
 * Given the set of WALs which have references in the metadata table, close any status messages with reference that WAL.
 *
 * @param conn
 *          Connector
 * @param closedWals
 *          {@link Set} of paths to WALs that marked as closed or unreferenced in zookeeper
 */
protected long updateReplicationEntries(Connector conn, Set<String> closedWals) {
    BatchScanner bs = null;
    BatchWriter bw = null;
    long recordsClosed = 0;
    try {
        bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
        bs = conn.createBatchScanner(MetadataTable.NAME, Authorizations.EMPTY, 4);
        bs.setRanges(Collections.singleton(Range.prefix(ReplicationSection.getRowPrefix())));
        bs.fetchColumnFamily(ReplicationSection.COLF);
        Text replFileText = new Text();
        for (Entry<Key, Value> entry : bs) {
            Status status;
            try {
                status = Status.parseFrom(entry.getValue().get());
            } catch (InvalidProtocolBufferException e) {
                log.error("Could not parse Status protobuf for {}", entry.getKey(), e);
                continue;
            }
            // Ignore things that aren't completely replicated as we can't delete those anyways
            MetadataSchema.ReplicationSection.getFile(entry.getKey(), replFileText);
            String replFile = replFileText.toString();
            boolean isClosed = closedWals.contains(replFile);
            // metadata doesn't have a reference to the given WAL
            if (!status.getClosed() && !replFile.endsWith(RFILE_SUFFIX) && isClosed) {
                try {
                    closeWal(bw, entry.getKey());
                    recordsClosed++;
                } catch (MutationsRejectedException e) {
                    log.error("Failed to submit delete mutation for {}", entry.getKey());
                    continue;
                }
            }
        }
    } catch (TableNotFoundException e) {
        log.error("Replication table was deleted", e);
    } finally {
        if (null != bs) {
            bs.close();
        }
        if (null != bw) {
            try {
                bw.close();
            } catch (MutationsRejectedException e) {
                log.error("Failed to write delete mutations for replication table", e);
            }
        }
    }
    return recordsClosed;
}