Examples with StatusMaker org.apache.accumulo.master.replication.StatusMaker used on opensource projects

Example 1 with StatusMaker

use of org.apache.accumulo.master.replication.StatusMaker in project accumulo by apache.

the class StatusMakerIT method orderRecordsCreatedWithNoCreatedTime.

@Test
public void orderRecordsCreatedWithNoCreatedTime() throws Exception {
    String sourceTable = testName.getMethodName();
    conn.tableOperations().create(sourceTable);
    ReplicationTableUtil.configureMetadataTable(conn, sourceTable);
    BatchWriter bw = conn.createBatchWriter(sourceTable, new BatchWriterConfig());
    String walPrefix = "hdfs://localhost:8020/accumulo/wals/tserver+port/";
    List<String> files = Arrays.asList(walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID());
    Map<String, Long> fileToTableId = new HashMap<>();
    Status.Builder statBuilder = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(true);
    Map<String, Long> statuses = new HashMap<>();
    long index = 1;
    for (String file : files) {
        Mutation m = new Mutation(ReplicationSection.getRowPrefix() + file);
        m.put(ReplicationSection.COLF, new Text(Long.toString(index)), ProtobufUtil.toValue(statBuilder.build()));
        bw.addMutation(m);
        fileToTableId.put(file, index);
        FileStatus status = EasyMock.mock(FileStatus.class);
        EasyMock.expect(status.getModificationTime()).andReturn(index);
        EasyMock.replay(status);
        statuses.put(file, index);
        EasyMock.expect(fs.exists(new Path(file))).andReturn(true);
        EasyMock.expect(fs.getFileStatus(new Path(file))).andReturn(status);
        index++;
    }
    EasyMock.replay(fs);
    bw.close();
    StatusMaker statusMaker = new StatusMaker(conn, fs);
    statusMaker.setSourceTableName(sourceTable);
    statusMaker.run();
    Scanner s = conn.createScanner(sourceTable, Authorizations.EMPTY);
    s.setRange(ReplicationSection.getRange());
    s.fetchColumnFamily(ReplicationSection.COLF);
    Assert.assertEquals(0, Iterables.size(s));
    s = ReplicationTable.getScanner(conn);
    OrderSection.limit(s);
    Iterator<Entry<Key, Value>> iter = s.iterator();
    Assert.assertTrue("Found no order records in replication table", iter.hasNext());
    Iterator<String> expectedFiles = files.iterator();
    Text buff = new Text();
    while (expectedFiles.hasNext() && iter.hasNext()) {
        String file = expectedFiles.next();
        Entry<Key, Value> entry = iter.next();
        Assert.assertEquals(file, OrderSection.getFile(entry.getKey(), buff));
        OrderSection.getTableId(entry.getKey(), buff);
        Assert.assertEquals(fileToTableId.get(file).intValue(), Integer.parseInt(buff.toString()));
        Status status = Status.parseFrom(entry.getValue().get());
        Assert.assertTrue(status.hasCreatedTime());
        Assert.assertEquals((long) statuses.get(file), status.getCreatedTime());
    }
    Assert.assertFalse("Found more files unexpectedly", expectedFiles.hasNext());
    Assert.assertFalse("Found more entries in replication table unexpectedly", iter.hasNext());
    s = conn.createScanner(sourceTable, Authorizations.EMPTY);
    s.setRange(ReplicationSection.getRange());
    s.fetchColumnFamily(ReplicationSection.COLF);
    Assert.assertEquals(0, Iterables.size(s));
    s = ReplicationTable.getScanner(conn);
    s.setRange(ReplicationSection.getRange());
    iter = s.iterator();
    Assert.assertTrue("Found no stat records in replication table", iter.hasNext());
    Collections.sort(files);
    expectedFiles = files.iterator();
    while (expectedFiles.hasNext() && iter.hasNext()) {
        String file = expectedFiles.next();
        Entry<Key, Value> entry = iter.next();
        Status status = Status.parseFrom(entry.getValue().get());
        Assert.assertTrue(status.hasCreatedTime());
        Assert.assertEquals((long) statuses.get(file), status.getCreatedTime());
    }
    Assert.assertFalse("Found more files unexpectedly", expectedFiles.hasNext());
    Assert.assertFalse("Found more entries in replication table unexpectedly", iter.hasNext());
}

Example 2 with StatusMaker

use of org.apache.accumulo.master.replication.StatusMaker in project accumulo by apache.

the class StatusMakerIT method closedMessagesCreateOrderRecords.

@Test
public void closedMessagesCreateOrderRecords() throws Exception {
    String sourceTable = testName.getMethodName();
    conn.tableOperations().create(sourceTable);
    ReplicationTableUtil.configureMetadataTable(conn, sourceTable);
    BatchWriter bw = conn.createBatchWriter(sourceTable, new BatchWriterConfig());
    String walPrefix = "hdfs://localhost:8020/accumulo/wals/tserver+port/";
    List<String> files = Arrays.asList(walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID());
    Map<String, Integer> fileToTableId = new HashMap<>();
    Status.Builder statBuilder = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(true);
    int index = 1;
    long time = System.currentTimeMillis();
    for (String file : files) {
        statBuilder.setCreatedTime(time++);
        Mutation m = new Mutation(ReplicationSection.getRowPrefix() + file);
        m.put(ReplicationSection.COLF, new Text(Integer.toString(index)), ProtobufUtil.toValue(statBuilder.build()));
        bw.addMutation(m);
        fileToTableId.put(file, index);
        index++;
    }
    bw.close();
    StatusMaker statusMaker = new StatusMaker(conn, fs);
    statusMaker.setSourceTableName(sourceTable);
    statusMaker.run();
    Iterator<Entry<Key, Value>> iter;
    Iterator<String> expectedFiles;
    try (Scanner s = conn.createScanner(sourceTable, Authorizations.EMPTY)) {
        s.setRange(ReplicationSection.getRange());
        s.fetchColumnFamily(ReplicationSection.COLF);
        Assert.assertEquals(0, Iterables.size(s));
    }
    try (Scanner s = ReplicationTable.getScanner(conn)) {
        OrderSection.limit(s);
        iter = s.iterator();
        Assert.assertTrue("Found no order records in replication table", iter.hasNext());
        expectedFiles = files.iterator();
        Text buff = new Text();
        while (expectedFiles.hasNext() && iter.hasNext()) {
            String file = expectedFiles.next();
            Entry<Key, Value> entry = iter.next();
            Assert.assertEquals(file, OrderSection.getFile(entry.getKey(), buff));
            OrderSection.getTableId(entry.getKey(), buff);
            Assert.assertEquals(fileToTableId.get(file).intValue(), Integer.parseInt(buff.toString()));
        }
    }
    Assert.assertFalse("Found more files unexpectedly", expectedFiles.hasNext());
    Assert.assertFalse("Found more entries in replication table unexpectedly", iter.hasNext());
}

Example 3 with StatusMaker

use of org.apache.accumulo.master.replication.StatusMaker in project accumulo by apache.

the class StatusMakerIT method closedMessagesAreDeleted.

@Test
public void closedMessagesAreDeleted() throws Exception {
    String sourceTable = testName.getMethodName();
    conn.tableOperations().create(sourceTable);
    ReplicationTableUtil.configureMetadataTable(conn, sourceTable);
    BatchWriter bw = conn.createBatchWriter(sourceTable, new BatchWriterConfig());
    String walPrefix = "hdfs://localhost:8020/accumulo/wals/tserver+port/";
    Set<String> files = Sets.newHashSet(walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID());
    Map<String, Integer> fileToTableId = new HashMap<>();
    Status stat = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(true).setCreatedTime(System.currentTimeMillis()).build();
    int index = 1;
    for (String file : files) {
        Mutation m = new Mutation(ReplicationSection.getRowPrefix() + file);
        m.put(ReplicationSection.COLF, new Text(Integer.toString(index)), ProtobufUtil.toValue(stat));
        bw.addMutation(m);
        fileToTableId.put(file, index);
        index++;
    }
    bw.close();
    StatusMaker statusMaker = new StatusMaker(conn, fs);
    statusMaker.setSourceTableName(sourceTable);
    statusMaker.run();
    try (Scanner s = conn.createScanner(sourceTable, Authorizations.EMPTY)) {
        s.setRange(ReplicationSection.getRange());
        s.fetchColumnFamily(ReplicationSection.COLF);
        for (Entry<Key, Value> e : s) {
            System.out.println(e.getKey().toStringNoTruncate() + " " + e.getValue());
        }
    }
    try (Scanner s = conn.createScanner(sourceTable, Authorizations.EMPTY)) {
        s.setRange(ReplicationSection.getRange());
        s.fetchColumnFamily(ReplicationSection.COLF);
        Assert.assertEquals(0, Iterables.size(s));
    }
}

Example 4 with StatusMaker

use of org.apache.accumulo.master.replication.StatusMaker in project accumulo by apache.

the class StatusMakerIT method statusRecordsCreated.

@Test
public void statusRecordsCreated() throws Exception {
    String sourceTable = testName.getMethodName();
    conn.tableOperations().create(sourceTable);
    ReplicationTableUtil.configureMetadataTable(conn, sourceTable);
    BatchWriter bw = conn.createBatchWriter(sourceTable, new BatchWriterConfig());
    String walPrefix = "hdfs://localhost:8020/accumulo/wals/tserver+port/";
    Set<String> files = Sets.newHashSet(walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID());
    Map<String, Integer> fileToTableId = new HashMap<>();
    int index = 1;
    long timeCreated = 0;
    Map<String, Long> fileToTimeCreated = new HashMap<>();
    for (String file : files) {
        Mutation m = new Mutation(ReplicationSection.getRowPrefix() + file);
        m.put(ReplicationSection.COLF, new Text(Integer.toString(index)), StatusUtil.fileCreatedValue(timeCreated));
        fileToTimeCreated.put(file, timeCreated);
        bw.addMutation(m);
        fileToTableId.put(file, index);
        index++;
        timeCreated++;
    }
    bw.close();
    StatusMaker statusMaker = new StatusMaker(conn, fs);
    statusMaker.setSourceTableName(sourceTable);
    statusMaker.run();
    try (Scanner s = ReplicationTable.getScanner(conn)) {
        StatusSection.limit(s);
        Text file = new Text();
        for (Entry<Key, Value> entry : s) {
            StatusSection.getFile(entry.getKey(), file);
            Table.ID tableId = StatusSection.getTableId(entry.getKey());
            Assert.assertTrue("Found unexpected file: " + file, files.contains(file.toString()));
            Assert.assertEquals(fileToTableId.get(file.toString()), new Integer(tableId.canonicalID()));
            timeCreated = fileToTimeCreated.get(file.toString());
            Assert.assertNotNull(timeCreated);
            Assert.assertEquals(StatusUtil.fileCreated(timeCreated), Status.parseFrom(entry.getValue().get()));
        }
    }
}

Example 5 with StatusMaker

use of org.apache.accumulo.master.replication.StatusMaker in project accumulo by apache.

the class StatusMakerIT method openMessagesAreNotDeleted.

@Test
public void openMessagesAreNotDeleted() throws Exception {
    String sourceTable = testName.getMethodName();
    conn.tableOperations().create(sourceTable);
    ReplicationTableUtil.configureMetadataTable(conn, sourceTable);
    BatchWriter bw = conn.createBatchWriter(sourceTable, new BatchWriterConfig());
    String walPrefix = "hdfs://localhost:8020/accumulo/wals/tserver+port/";
    Set<String> files = Sets.newHashSet(walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID());
    Map<String, Integer> fileToTableId = new HashMap<>();
    int index = 1;
    long timeCreated = 0;
    for (String file : files) {
        Mutation m = new Mutation(ReplicationSection.getRowPrefix() + file);
        m.put(ReplicationSection.COLF, new Text(Integer.toString(index)), StatusUtil.fileCreatedValue(timeCreated));
        bw.addMutation(m);
        fileToTableId.put(file, index);
        index++;
        timeCreated++;
    }
    bw.close();
    StatusMaker statusMaker = new StatusMaker(conn, fs);
    statusMaker.setSourceTableName(sourceTable);
    statusMaker.run();
    try (Scanner s = conn.createScanner(sourceTable, Authorizations.EMPTY)) {
        s.setRange(ReplicationSection.getRange());
        s.fetchColumnFamily(ReplicationSection.COLF);
        Assert.assertEquals(files.size(), Iterables.size(s));
    }
}