Examples with PreparedMutations org.apache.accumulo.tserver.tablet.PreparedMutations used on opensource projects

Example 1 with PreparedMutations

use of org.apache.accumulo.tserver.tablet.PreparedMutations in project accumulo by apache.

the class ThriftClientHandler method writeConditionalMutations.

private void writeConditionalMutations(Map<KeyExtent, List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession sess) {
    Set<Entry<KeyExtent, List<ServerConditionalMutation>>> es = updates.entrySet();
    Map<CommitSession, List<Mutation>> sendables = new HashMap<>();
    Map<CommitSession, TabletMutations> loggables = new HashMap<>();
    boolean sessionCanceled = sess.interruptFlag.get();
    Span span = TraceUtil.startSpan(this.getClass(), "writeConditionalMutations::prep");
    try (Scope scope = span.makeCurrent()) {
        long t1 = System.currentTimeMillis();
        for (Entry<KeyExtent, List<ServerConditionalMutation>> entry : es) {
            final Tablet tablet = server.getOnlineTablet(entry.getKey());
            if (tablet == null || tablet.isClosed() || sessionCanceled) {
                addMutationsAsTCMResults(results, entry.getValue(), TCMStatus.IGNORED);
            } else {
                final Durability durability = DurabilityImpl.resolveDurabilty(sess.durability, tablet.getDurability());
                @SuppressWarnings("unchecked") List<Mutation> mutations = (List<Mutation>) (List<? extends Mutation>) entry.getValue();
                if (!mutations.isEmpty()) {
                    PreparedMutations prepared = tablet.prepareMutationsForCommit(new TservConstraintEnv(server.getContext(), security, sess.credentials), mutations);
                    if (prepared.tabletClosed()) {
                        addMutationsAsTCMResults(results, mutations, TCMStatus.IGNORED);
                    } else {
                        if (!prepared.getNonViolators().isEmpty()) {
                            // Only log and commit mutations that did not violate constraints.
                            List<Mutation> validMutations = prepared.getNonViolators();
                            addMutationsAsTCMResults(results, validMutations, TCMStatus.ACCEPTED);
                            CommitSession session = prepared.getCommitSession();
                            if (durability != Durability.NONE) {
                                loggables.put(session, new TabletMutations(session, validMutations, durability));
                            }
                            sendables.put(session, validMutations);
                        }
                        if (!prepared.getViolators().isEmpty()) {
                            addMutationsAsTCMResults(results, prepared.getViolators(), TCMStatus.VIOLATED);
                        }
                    }
                }
            }
        }
        long t2 = System.currentTimeMillis();
        updateAvgPrepTime(t2 - t1, es.size());
    } catch (Exception e) {
        TraceUtil.setException(span, e, true);
        throw e;
    } finally {
        span.end();
    }
    Span span2 = TraceUtil.startSpan(this.getClass(), "writeConditionalMutations::wal");
    try (Scope scope = span2.makeCurrent()) {
        while (!loggables.isEmpty()) {
            try {
                long t1 = System.currentTimeMillis();
                server.logger.logManyTablets(loggables);
                long t2 = System.currentTimeMillis();
                updateWalogWriteTime(t2 - t1);
                break;
            } catch (IOException | FSError ex) {
                TraceUtil.setException(span2, ex, false);
                log.warn("logging mutations failed, retrying");
            } catch (Exception t) {
                log.error("Unknown exception logging mutations, counts for" + " mutations in flight not decremented!", t);
                throw new RuntimeException(t);
            }
        }
    } catch (Exception e) {
        TraceUtil.setException(span2, e, true);
        throw e;
    } finally {
        span2.end();
    }
    Span span3 = TraceUtil.startSpan(this.getClass(), "writeConditionalMutations::commit");
    try (Scope scope = span3.makeCurrent()) {
        long t1 = System.currentTimeMillis();
        sendables.forEach(CommitSession::commit);
        long t2 = System.currentTimeMillis();
        updateAvgCommitTime(t2 - t1, sendables.size());
    } catch (Exception e) {
        TraceUtil.setException(span3, e, true);
        throw e;
    } finally {
        span3.end();
    }
}

Example 2 with PreparedMutations

use of org.apache.accumulo.tserver.tablet.PreparedMutations in project accumulo by apache.

the class ThriftClientHandler method flush.

private void flush(UpdateSession us) {
    int mutationCount = 0;
    Map<CommitSession, List<Mutation>> sendables = new HashMap<>();
    Map<CommitSession, TabletMutations> loggables = new HashMap<>();
    Throwable error = null;
    long pt1 = System.currentTimeMillis();
    boolean containsMetadataTablet = false;
    for (Tablet tablet : us.queuedMutations.keySet()) {
        if (tablet.getExtent().isMeta()) {
            containsMetadataTablet = true;
        }
    }
    if (!containsMetadataTablet && !us.queuedMutations.isEmpty()) {
        server.resourceManager.waitUntilCommitsAreEnabled();
    }
    Span span = TraceUtil.startSpan(this.getClass(), "flush::prep");
    try (Scope scope = span.makeCurrent()) {
        for (Entry<Tablet, ? extends List<Mutation>> entry : us.queuedMutations.entrySet()) {
            Tablet tablet = entry.getKey();
            Durability durability = DurabilityImpl.resolveDurabilty(us.durability, tablet.getDurability());
            List<Mutation> mutations = entry.getValue();
            if (!mutations.isEmpty()) {
                try {
                    server.updateMetrics.addMutationArraySize(mutations.size());
                    PreparedMutations prepared = tablet.prepareMutationsForCommit(us.cenv, mutations);
                    if (prepared.tabletClosed()) {
                        if (us.currentTablet == tablet) {
                            us.currentTablet = null;
                        }
                        us.failures.put(tablet.getExtent(), us.successfulCommits.get(tablet));
                    } else {
                        if (!prepared.getNonViolators().isEmpty()) {
                            List<Mutation> validMutations = prepared.getNonViolators();
                            CommitSession session = prepared.getCommitSession();
                            if (durability != Durability.NONE) {
                                loggables.put(session, new TabletMutations(session, validMutations, durability));
                            }
                            sendables.put(session, validMutations);
                        }
                        if (!prepared.getViolations().isEmpty()) {
                            us.violations.add(prepared.getViolations());
                            server.updateMetrics.addConstraintViolations(0);
                        }
                        // Use the size of the original mutation list, regardless of how many mutations
                        // did not violate constraints.
                        mutationCount += mutations.size();
                    }
                } catch (Exception t) {
                    error = t;
                    log.error("Unexpected error preparing for commit", error);
                    TraceUtil.setException(span, t, false);
                    break;
                }
            }
        }
    } catch (Exception e) {
        TraceUtil.setException(span, e, true);
        throw e;
    } finally {
        span.end();
    }
    long pt2 = System.currentTimeMillis();
    us.prepareTimes.addStat(pt2 - pt1);
    updateAvgPrepTime(pt2 - pt1, us.queuedMutations.size());
    if (error != null) {
        sendables.forEach((commitSession, value) -> commitSession.abortCommit());
        throw new RuntimeException(error);
    }
    try {
        Span span2 = TraceUtil.startSpan(this.getClass(), "flush::wal");
        try (Scope scope = span2.makeCurrent()) {
            while (true) {
                try {
                    long t1 = System.currentTimeMillis();
                    server.logger.logManyTablets(loggables);
                    long t2 = System.currentTimeMillis();
                    us.walogTimes.addStat(t2 - t1);
                    updateWalogWriteTime((t2 - t1));
                    break;
                } catch (IOException | FSError ex) {
                    log.warn("logging mutations failed, retrying");
                } catch (Exception t) {
                    log.error("Unknown exception logging mutations, counts" + " for mutations in flight not decremented!", t);
                    throw new RuntimeException(t);
                }
            }
        } catch (Exception e) {
            TraceUtil.setException(span2, e, true);
            throw e;
        } finally {
            span2.end();
        }
        Span span3 = TraceUtil.startSpan(this.getClass(), "flush::commit");
        try (Scope scope = span3.makeCurrent()) {
            long t1 = System.currentTimeMillis();
            sendables.forEach((commitSession, mutations) -> {
                commitSession.commit(mutations);
                KeyExtent extent = commitSession.getExtent();
                if (us.currentTablet != null && extent == us.currentTablet.getExtent()) {
                    // because constraint violations may filter out some
                    // mutations, for proper accounting with the client code,
                    // need to increment the count based on the original
                    // number of mutations from the client NOT the filtered number
                    us.successfulCommits.increment(us.currentTablet, us.queuedMutations.get(us.currentTablet).size());
                }
            });
            long t2 = System.currentTimeMillis();
            us.flushTime += (t2 - pt1);
            us.commitTimes.addStat(t2 - t1);
            updateAvgCommitTime(t2 - t1, sendables.size());
        } finally {
            span3.end();
        }
    } finally {
        us.queuedMutations.clear();
        if (us.currentTablet != null) {
            us.queuedMutations.put(us.currentTablet, new ArrayList<>());
        }
        server.updateTotalQueuedMutationSize(-us.queuedMutationSize);
        us.queuedMutationSize = 0;
    }
    us.totalUpdates += mutationCount;
}

Example 3 with PreparedMutations

use of org.apache.accumulo.tserver.tablet.PreparedMutations in project accumulo by apache.

the class ThriftClientHandler method update.

@Override
public void update(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, TMutation tmutation, TDurability tdurability) throws NotServingTabletException, ConstraintViolationException, ThriftSecurityException {
    final TableId tableId = TableId.of(new String(tkeyExtent.getTable(), UTF_8));
    NamespaceId namespaceId = getNamespaceId(credentials, tableId);
    if (!security.canWrite(credentials, tableId, namespaceId)) {
        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
    }
    final KeyExtent keyExtent = KeyExtent.fromThrift(tkeyExtent);
    final Tablet tablet = server.getOnlineTablet(KeyExtent.copyOf(keyExtent));
    if (tablet == null) {
        throw new NotServingTabletException(tkeyExtent);
    }
    Durability tabletDurability = tablet.getDurability();
    if (!keyExtent.isMeta()) {
        try {
            server.resourceManager.waitUntilCommitsAreEnabled();
        } catch (HoldTimeoutException hte) {
            // was a failure and it should retry.
            throw new NotServingTabletException(tkeyExtent);
        }
    }
    final long opid = writeTracker.startWrite(TabletType.type(keyExtent));
    try {
        final Mutation mutation = new ServerMutation(tmutation);
        final List<Mutation> mutations = Collections.singletonList(mutation);
        PreparedMutations prepared;
        Span span = TraceUtil.startSpan(this.getClass(), "update::prep");
        try (Scope scope = span.makeCurrent()) {
            prepared = tablet.prepareMutationsForCommit(new TservConstraintEnv(server.getContext(), security, credentials), mutations);
        } catch (Exception e) {
            TraceUtil.setException(span, e, true);
            throw e;
        } finally {
            span.end();
        }
        if (prepared.tabletClosed()) {
            throw new NotServingTabletException(tkeyExtent);
        } else if (!prepared.getViolators().isEmpty()) {
            throw new ConstraintViolationException(prepared.getViolations().asList().stream().map(ConstraintViolationSummary::toThrift).collect(Collectors.toList()));
        } else {
            CommitSession session = prepared.getCommitSession();
            Durability durability = DurabilityImpl.resolveDurabilty(DurabilityImpl.fromThrift(tdurability), tabletDurability);
            // Instead of always looping on true, skip completely when durability is NONE.
            while (durability != Durability.NONE) {
                try {
                    Span span2 = TraceUtil.startSpan(this.getClass(), "update::wal");
                    try (Scope scope = span2.makeCurrent()) {
                        server.logger.log(session, mutation, durability);
                    } catch (Exception e) {
                        TraceUtil.setException(span2, e, true);
                        throw e;
                    } finally {
                        span2.end();
                    }
                    break;
                } catch (IOException ex) {
                    log.warn("Error writing mutations to log", ex);
                }
            }
            Span span3 = TraceUtil.startSpan(this.getClass(), "update::commit");
            try (Scope scope = span3.makeCurrent()) {
                session.commit(mutations);
            } catch (Exception e) {
                TraceUtil.setException(span3, e, true);
                throw e;
            } finally {
                span3.end();
            }
        }
    } finally {
        writeTracker.finishWrite(opid);
    }
}