Examples with TMutation org.apache.accumulo.core.dataImpl.thrift.TMutation used on opensource projects

Example 6 with TMutation

use of org.apache.accumulo.core.dataImpl.thrift.TMutation in project accumulo by apache.

the class BatchWriterReplicationReplayerTest method replicationSourcesArePreserved.

@Test
public void replicationSourcesArePreserved() throws Exception {
    final BatchWriterReplicationReplayer replayer = new BatchWriterReplicationReplayer();
    final String tableName = "foo";
    final long systemTimestamp = 1000;
    final String peerName = "peer";
    final BatchWriterConfig bwCfg = new BatchWriterConfig();
    bwCfg.setMaxMemory(1L);
    LogFileKey key = new LogFileKey();
    key.event = LogEvents.MANY_MUTATIONS;
    key.seq = 1;
    key.tabletId = 1;
    WalEdits edits = new WalEdits();
    // Make a mutation without timestamps
    Mutation m = new Mutation("row");
    m.put("cf", "cq1", "value");
    m.put("cf", "cq2", "value");
    m.put("cf", "cq3", "value");
    m.put("cf", "cq4", "value");
    m.put("cf", "cq5", "value");
    // This Mutation "came" from a system called "peer"
    m.addReplicationSource(peerName);
    // Make it a TMutation
    TMutation tMutation = m.toThrift();
    // And then make a ServerMutation from the TMutation, adding in our systemTimestamp
    ServerMutation sMutation = new ServerMutation(tMutation);
    sMutation.setSystemTimestamp(systemTimestamp);
    // Serialize the ServerMutation (what AccumuloReplicaSystem will be doing)
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(baos);
    key.write(out);
    out.writeInt(1);
    sMutation.write(out);
    out.close();
    // Add it to our "input" to BatchWriterReplicationReplayer
    edits.addToEdits(ByteBuffer.wrap(baos.toByteArray()));
    Mutation expectedMutation = new Mutation("row");
    expectedMutation.put("cf", "cq1", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq2", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq3", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq4", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq5", sMutation.getSystemTimestamp(), "value");
    // We expect our peer name to be preserved in the mutation that gets written
    expectedMutation.addReplicationSource(peerName);
    expect(conf.getAsBytes(Property.TSERV_REPLICATION_BW_REPLAYER_MEMORY)).andReturn(bwCfg.getMaxMemory());
    expect(context.createBatchWriter(tableName, bwCfg)).andReturn(bw);
    bw.addMutations(Lists.newArrayList(expectedMutation));
    expectLastCall().once();
    bw.close();
    expectLastCall().once();
    replay(context, conf, bw);
    replayer.replicateLog(context, tableName, edits);
}

Example 7 with TMutation

use of org.apache.accumulo.core.dataImpl.thrift.TMutation in project accumulo by apache.

the class ThriftClientHandler method update.

@Override
public void update(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, TMutation tmutation, TDurability tdurability) throws NotServingTabletException, ConstraintViolationException, ThriftSecurityException {
    final TableId tableId = TableId.of(new String(tkeyExtent.getTable(), UTF_8));
    NamespaceId namespaceId = getNamespaceId(credentials, tableId);
    if (!security.canWrite(credentials, tableId, namespaceId)) {
        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
    }
    final KeyExtent keyExtent = KeyExtent.fromThrift(tkeyExtent);
    final Tablet tablet = server.getOnlineTablet(KeyExtent.copyOf(keyExtent));
    if (tablet == null) {
        throw new NotServingTabletException(tkeyExtent);
    }
    Durability tabletDurability = tablet.getDurability();
    if (!keyExtent.isMeta()) {
        try {
            server.resourceManager.waitUntilCommitsAreEnabled();
        } catch (HoldTimeoutException hte) {
            // was a failure and it should retry.
            throw new NotServingTabletException(tkeyExtent);
        }
    }
    final long opid = writeTracker.startWrite(TabletType.type(keyExtent));
    try {
        final Mutation mutation = new ServerMutation(tmutation);
        final List<Mutation> mutations = Collections.singletonList(mutation);
        PreparedMutations prepared;
        Span span = TraceUtil.startSpan(this.getClass(), "update::prep");
        try (Scope scope = span.makeCurrent()) {
            prepared = tablet.prepareMutationsForCommit(new TservConstraintEnv(server.getContext(), security, credentials), mutations);
        } catch (Exception e) {
            TraceUtil.setException(span, e, true);
            throw e;
        } finally {
            span.end();
        }
        if (prepared.tabletClosed()) {
            throw new NotServingTabletException(tkeyExtent);
        } else if (!prepared.getViolators().isEmpty()) {
            throw new ConstraintViolationException(prepared.getViolations().asList().stream().map(ConstraintViolationSummary::toThrift).collect(Collectors.toList()));
        } else {
            CommitSession session = prepared.getCommitSession();
            Durability durability = DurabilityImpl.resolveDurabilty(DurabilityImpl.fromThrift(tdurability), tabletDurability);
            // Instead of always looping on true, skip completely when durability is NONE.
            while (durability != Durability.NONE) {
                try {
                    Span span2 = TraceUtil.startSpan(this.getClass(), "update::wal");
                    try (Scope scope = span2.makeCurrent()) {
                        server.logger.log(session, mutation, durability);
                    } catch (Exception e) {
                        TraceUtil.setException(span2, e, true);
                        throw e;
                    } finally {
                        span2.end();
                    }
                    break;
                } catch (IOException ex) {
                    log.warn("Error writing mutations to log", ex);
                }
            }
            Span span3 = TraceUtil.startSpan(this.getClass(), "update::commit");
            try (Scope scope = span3.makeCurrent()) {
                session.commit(mutations);
            } catch (Exception e) {
                TraceUtil.setException(span3, e, true);
                throw e;
            } finally {
                span3.end();
            }
        }
    } finally {
        writeTracker.finishWrite(opid);
    }
}

Example 8 with TMutation

use of org.apache.accumulo.core.dataImpl.thrift.TMutation in project accumulo by apache.

the class ThriftClientHandler method applyUpdates.

@Override
public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent tkeyExtent, List<TMutation> tmutations) {
    UpdateSession us = (UpdateSession) server.sessionManager.reserveSession(updateID);
    if (us == null) {
        return;
    }
    boolean reserved = true;
    try {
        KeyExtent keyExtent = KeyExtent.fromThrift(tkeyExtent);
        setUpdateTablet(us, keyExtent);
        if (us.currentTablet != null) {
            long additionalMutationSize = 0;
            List<Mutation> mutations = us.queuedMutations.get(us.currentTablet);
            for (TMutation tmutation : tmutations) {
                Mutation mutation = new ServerMutation(tmutation);
                mutations.add(mutation);
                additionalMutationSize += mutation.numBytes();
            }
            us.queuedMutationSize += additionalMutationSize;
            long totalQueued = server.updateTotalQueuedMutationSize(additionalMutationSize);
            long total = server.getConfiguration().getAsBytes(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX);
            if (totalQueued > total) {
                try {
                    flush(us);
                } catch (HoldTimeoutException hte) {
                    // Assumption is that the client has timed out and is gone. If that's not the case,
                    // then removing the session should cause the client to fail
                    // in such a way that it retries.
                    log.debug("HoldTimeoutException during applyUpdates, removing session");
                    server.sessionManager.removeSession(updateID, true);
                    reserved = false;
                }
            }
        }
    } finally {
        if (reserved) {
            server.sessionManager.unreserveSession(us);
        }
    }
}