Examples with Status org.apache.accumulo.server.replication.proto.Replication.Status used on opensource projects

Example 56 with Status

use of org.apache.accumulo.server.replication.proto.Replication.Status in project accumulo by apache.

the class StatusCombinerTest method newStatusWithNewIngestSingleBuilder.

@Test
public void newStatusWithNewIngestSingleBuilder() {
    Status orig = StatusUtil.fileCreated(100);
    Status status = StatusUtil.replicatedAndIngested(builder, 10, 20);
    Status ret = combiner.typedReduce(key, Arrays.asList(orig, status).iterator());
    Assert.assertEquals(10l, ret.getBegin());
    Assert.assertEquals(20l, ret.getEnd());
    Assert.assertEquals(100l, ret.getCreatedTime());
    Assert.assertEquals(false, ret.getClosed());
}

Example 57 with Status

use of org.apache.accumulo.server.replication.proto.Replication.Status in project accumulo by apache.

the class StatusCombinerTest method commutativeNewUpdatesSingleBuilder.

@Test
public void commutativeNewUpdatesSingleBuilder() {
    Status newFile = StatusUtil.fileCreated(100), firstSync = StatusUtil.ingestedUntil(builder, 100), secondSync = StatusUtil.ingestedUntil(builder, 200);
    Status order1 = combiner.typedReduce(key, Arrays.asList(newFile, firstSync, secondSync).iterator()), order2 = combiner.typedReduce(key, Arrays.asList(newFile, secondSync, firstSync).iterator());
    Assert.assertEquals(order1, order2);
}

Example 58 with Status

use of org.apache.accumulo.server.replication.proto.Replication.Status in project accumulo by apache.

the class StatusMaker method setAndGetCreatedTime.

private long setAndGetCreatedTime(Path file, String tableId) throws IOException, MutationsRejectedException {
    long createdTime;
    if (fs.exists(file)) {
        createdTime = fs.getFileStatus(file).getModificationTime();
    } else {
        createdTime = System.currentTimeMillis();
    }
    Status status = Status.newBuilder().setCreatedTime(createdTime).build();
    Mutation m = new Mutation(new Text(ReplicationSection.getRowPrefix() + file.toString()));
    m.put(MetadataSchema.ReplicationSection.COLF, new Text(tableId), ProtobufUtil.toValue(status));
    replicationWriter.addMutation(m);
    replicationWriter.flush();
    return createdTime;
}

Example 59 with Status

use of org.apache.accumulo.server.replication.proto.Replication.Status in project accumulo by apache.

the class ReplicationProcessor method process.

@Override
public void process(String workID, byte[] data) {
    ReplicationTarget target = DistributedWorkQueueWorkAssignerHelper.fromQueueKey(workID).getValue();
    String file = new String(data, UTF_8);
    log.debug("Received replication work for {} to {}", file, target);
    ReplicaSystem replica;
    try {
        replica = getReplicaSystem(target);
    } catch (Exception e) {
        log.error("Could not instantiate ReplicaSystem for {}, waiting before returning the work", target, e);
        try {
            // TODO configurable
            Thread.sleep(5000);
        } catch (InterruptedException ie) {
            Thread.currentThread().interrupt();
        }
        return;
    }
    Status status;
    try {
        status = getStatus(file, target);
    } catch (ReplicationTableOfflineException | AccumuloException | AccumuloSecurityException e) {
        log.error("Could not look for replication record", e);
        throw new IllegalStateException("Could not look for replication record", e);
    } catch (InvalidProtocolBufferException e) {
        log.error("Could not deserialize Status from Work section for {} and ", file, target);
        throw new RuntimeException("Could not parse Status for work record", e);
    } catch (NoSuchElementException e) {
        log.error("Assigned work for {} to {} but could not find work record", file, target);
        return;
    }
    log.debug("Current status for {} replicating to {}: {}", file, target, ProtobufUtil.toString(status));
    // We don't need to do anything (shouldn't have gotten this work record in the first place)
    if (!StatusUtil.isWorkRequired(status)) {
        log.info("Received work request for {} and {}, but it does not need replication. Ignoring...", file, target);
        return;
    }
    // Sanity check that nothing bad happened and our replication source still exists
    Path filePath = new Path(file);
    try {
        if (!doesFileExist(filePath, target)) {
            return;
        }
    } catch (IOException e) {
        log.error("Could not determine if file exists {}", filePath, e);
        throw new RuntimeException(e);
    }
    log.debug("Replicating {} to {} using {}", filePath, target, replica.getClass().getName());
    Status newStatus = replica.replicate(filePath, status, target, getHelper());
    log.debug("Finished replicating {}. Original status: {}, New status: {}", filePath, status, newStatus);
}

Example 60 with Status

use of org.apache.accumulo.server.replication.proto.Replication.Status in project accumulo by apache.

the class AccumuloReplicaSystemTest method onlyChooseMutationsForDesiredTableWithClosedStatus.

@Test
public void onlyChooseMutationsForDesiredTableWithClosedStatus() throws Exception {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);
    LogFileKey key = new LogFileKey();
    LogFileValue value = new LogFileValue();
    // What is seq used for?
    key.seq = 1l;
    /*
     * Disclaimer: the following series of LogFileKey and LogFileValue pairs have *no* bearing whatsoever in reality regarding what these entries would actually
     * look like in a WAL. They are solely for testing that each LogEvents is handled, order is not important.
     */
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
    key.tid = 1;
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("2"), null, null);
    key.tid = 2;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.OPEN;
    key.tid = LogFileKey.VERSION;
    key.tserverSession = "foobar";
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("badrow")));
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_START;
    key.tid = 2;
    key.filename = "/accumulo/tables/1/t-000001/A000001.rf";
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
    key.tid = 3;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_FINISH;
    key.tid = 6;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.tid = 3;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    dos.close();
    Map<String, String> confMap = new HashMap<>();
    confMap.put(Property.REPLICATION_NAME.getKey(), "source");
    AccumuloConfiguration conf = new ConfigurationCopy(confMap);
    AccumuloReplicaSystem ars = new AccumuloReplicaSystem();
    ars.setConf(conf);
    // Setting the file to be closed with the infinite end implies that we need to bump the begin up to Long.MAX_VALUE
    // If it were still open, more data could be appended that we need to process
    Status status = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(true).build();
    DataInputStream dis = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
    WalReplication repl = ars.getWalEdits(new ReplicationTarget("peer", "1", Table.ID.of("1")), dis, new Path("/accumulo/wals/tserver+port/wal"), status, Long.MAX_VALUE, new HashSet<>());
    // We stopped because we got to the end of the file
    Assert.assertEquals(Long.MAX_VALUE, repl.entriesConsumed);
    Assert.assertEquals(2, repl.walEdits.getEditsSize());
    Assert.assertEquals(2, repl.sizeInRecords);
    Assert.assertNotEquals(0, repl.sizeInBytes);
}