Examples with Tablet org.apache.accumulo.tserver.tablet.Tablet used on opensource projects

Example 1 with Tablet

use of org.apache.accumulo.tserver.tablet.Tablet in project accumulo by apache.

the class TabletServer method splitTablet.

private TreeMap<KeyExtent, TabletData> splitTablet(Tablet tablet, byte[] splitPoint) throws IOException {
    long t1 = System.currentTimeMillis();
    TreeMap<KeyExtent, TabletData> tabletInfo = tablet.split(splitPoint);
    if (tabletInfo == null) {
        return null;
    }
    log.info("Starting split: {}", tablet.getExtent());
    statsKeeper.incrementStatusSplit();
    long start = System.currentTimeMillis();
    Tablet[] newTablets = new Tablet[2];
    Entry<KeyExtent, TabletData> first = tabletInfo.firstEntry();
    TabletResourceManager newTrm0 = resourceManager.createTabletResourceManager(first.getKey(), getTableConfiguration(first.getKey()));
    newTablets[0] = new Tablet(TabletServer.this, first.getKey(), newTrm0, first.getValue());
    Entry<KeyExtent, TabletData> last = tabletInfo.lastEntry();
    TabletResourceManager newTrm1 = resourceManager.createTabletResourceManager(last.getKey(), getTableConfiguration(last.getKey()));
    newTablets[1] = new Tablet(TabletServer.this, last.getKey(), newTrm1, last.getValue());
    // roll tablet stats over into tablet server's statsKeeper object as
    // historical data
    statsKeeper.saveMajorMinorTimes(tablet.getTabletStats());
    // lose the reference to the old tablet and open two new ones
    synchronized (onlineTablets) {
        onlineTablets.remove(tablet.getExtent());
        onlineTablets.put(newTablets[0].getExtent(), newTablets[0]);
        onlineTablets.put(newTablets[1].getExtent(), newTablets[1]);
    }
    // tell the master
    enqueueMasterMessage(new SplitReportMessage(tablet.getExtent(), newTablets[0].getExtent(), new Text("/" + newTablets[0].getLocation().getName()), newTablets[1].getExtent(), new Text("/" + newTablets[1].getLocation().getName())));
    statsKeeper.updateTime(Operation.SPLIT, start, 0, false);
    long t2 = System.currentTimeMillis();
    log.info("Tablet split: {} size0 {} size1 {} time {}ms", tablet.getExtent(), newTablets[0].estimateTabletSize(), newTablets[1].estimateTabletSize(), (t2 - t1));
    return tabletInfo;
}

Example 2 with Tablet

use of org.apache.accumulo.tserver.tablet.Tablet in project accumulo by apache.

the class TabletServer method config.

public void config(String hostname) {
    log.info("Tablet server starting on {}", hostname);
    majorCompactorThread = new Daemon(new LoggingRunnable(log, new MajorCompactor(getConfiguration())));
    majorCompactorThread.setName("Split/MajC initiator");
    majorCompactorThread.start();
    clientAddress = HostAndPort.fromParts(hostname, 0);
    try {
        AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig() {

            @Override
            public Map<String, String> getVfsContextClasspathProperties() {
                return getConfiguration().getAllPropertiesWithPrefix(Property.VFS_CONTEXT_CLASSPATH_PROPERTY);
            }
        });
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    // A task that cleans up unused classloader contexts
    Runnable contextCleaner = new Runnable() {

        @Override
        public void run() {
            Set<String> contextProperties = getServerConfigurationFactory().getSystemConfiguration().getAllPropertiesWithPrefix(Property.VFS_CONTEXT_CLASSPATH_PROPERTY).keySet();
            Set<String> configuredContexts = new HashSet<>();
            for (String prop : contextProperties) {
                configuredContexts.add(prop.substring(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.name().length()));
            }
            try {
                AccumuloVFSClassLoader.getContextManager().removeUnusedContexts(configuredContexts);
            } catch (IOException e) {
                log.warn("{}", e.getMessage(), e);
            }
        }
    };
    AccumuloConfiguration aconf = getConfiguration();
    SimpleTimer.getInstance(aconf).schedule(contextCleaner, 60000, 60000);
    FileSystemMonitor.start(aconf, Property.TSERV_MONITOR_FS);
    Runnable gcDebugTask = new Runnable() {

        @Override
        public void run() {
            gcLogger.logGCInfo(getConfiguration());
        }
    };
    SimpleTimer.getInstance(aconf).schedule(gcDebugTask, 0, TIME_BETWEEN_GC_CHECKS);
    Runnable constraintTask = new Runnable() {

        @Override
        public void run() {
            ArrayList<Tablet> tablets;
            synchronized (onlineTablets) {
                tablets = new ArrayList<>(onlineTablets.values());
            }
            for (Tablet tablet : tablets) {
                tablet.checkConstraints();
            }
        }
    };
    SimpleTimer.getInstance(aconf).schedule(constraintTask, 0, 1000);
}

Example 3 with Tablet

use of org.apache.accumulo.tserver.tablet.Tablet in project accumulo by apache.

the class TabletServerMetricsUtil method getAverageFilesPerTablet.

public double getAverageFilesPerTablet() {
    int count = 0;
    long result = 0;
    for (Tablet tablet : tserver.getOnlineTablets()) {
        result += tablet.getDatafiles().size();
        count++;
    }
    if (count == 0)
        return 0;
    return result / (double) count;
}

Example 4 with Tablet

use of org.apache.accumulo.tserver.tablet.Tablet in project accumulo by apache.

the class LookupTask method run.

@Override
public void run() {
    MultiScanSession session = (MultiScanSession) server.getSession(scanID);
    String oldThreadName = Thread.currentThread().getName();
    try {
        if (isCancelled() || session == null)
            return;
        TableConfiguration acuTableConf = server.getTableConfiguration(session.threadPoolExtent);
        long maxResultsSize = acuTableConf.getAsBytes(Property.TABLE_SCAN_MAXMEM);
        runState.set(ScanRunState.RUNNING);
        Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Table: ");
        long bytesAdded = 0;
        long maxScanTime = 4000;
        long startTime = System.currentTimeMillis();
        List<KVEntry> results = new ArrayList<>();
        Map<KeyExtent, List<Range>> failures = new HashMap<>();
        List<KeyExtent> fullScans = new ArrayList<>();
        KeyExtent partScan = null;
        Key partNextKey = null;
        boolean partNextKeyInclusive = false;
        Iterator<Entry<KeyExtent, List<Range>>> iter = session.queries.entrySet().iterator();
        // check the time so that the read ahead thread is not monopolized
        while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
            Entry<KeyExtent, List<Range>> entry = iter.next();
            iter.remove();
            // check that tablet server is serving requested tablet
            Tablet tablet = server.getOnlineTablet(entry.getKey());
            if (tablet == null) {
                failures.put(entry.getKey(), entry.getValue());
                continue;
            }
            Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Tablet: " + entry.getKey().toString());
            LookupResult lookupResult;
            try {
                // canceled
                if (isCancelled())
                    interruptFlag.set(true);
                lookupResult = tablet.lookup(entry.getValue(), session.columnSet, session.auths, results, maxResultsSize - bytesAdded, session.ssiList, session.ssio, interruptFlag, session.samplerConfig, session.batchTimeOut, session.context);
                // if the tablet was closed it it possible that the
                // interrupt flag was set.... do not want it set for
                // the next
                // lookup
                interruptFlag.set(false);
            } catch (IOException e) {
                log.warn("lookup failed for tablet " + entry.getKey(), e);
                throw new RuntimeException(e);
            }
            bytesAdded += lookupResult.bytesAdded;
            if (lookupResult.unfinishedRanges.size() > 0) {
                if (lookupResult.closed) {
                    failures.put(entry.getKey(), lookupResult.unfinishedRanges);
                } else {
                    session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
                    partScan = entry.getKey();
                    partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
                    partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
                }
            } else {
                fullScans.add(entry.getKey());
            }
        }
        long finishTime = System.currentTimeMillis();
        session.totalLookupTime += (finishTime - startTime);
        session.numEntries += results.size();
        // convert everything to thrift before adding result
        List<TKeyValue> retResults = new ArrayList<>();
        for (KVEntry entry : results) retResults.add(new TKeyValue(entry.getKey().toThrift(), ByteBuffer.wrap(entry.getValue().get())));
        Map<TKeyExtent, List<TRange>> retFailures = Translator.translate(failures, Translators.KET, new Translator.ListTranslator<>(Translators.RT));
        List<TKeyExtent> retFullScans = Translator.translate(fullScans, Translators.KET);
        TKeyExtent retPartScan = null;
        TKey retPartNextKey = null;
        if (partScan != null) {
            retPartScan = partScan.toThrift();
            retPartNextKey = partNextKey.toThrift();
        }
        // add results to queue
        addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, session.queries.size() != 0));
    } catch (IterationInterruptedException iie) {
        if (!isCancelled()) {
            log.warn("Iteration interrupted, when scan not cancelled", iie);
            addResult(iie);
        }
    } catch (SampleNotPresentException e) {
        addResult(e);
    } catch (Throwable e) {
        log.warn("exception while doing multi-scan ", e);
        addResult(e);
    } finally {
        Thread.currentThread().setName(oldThreadName);
        runState.set(ScanRunState.FINISHED);
    }
}

Example 5 with Tablet

use of org.apache.accumulo.tserver.tablet.Tablet in project accumulo by apache.

the class NextBatchTask method run.

@Override
public void run() {
    final ScanSession scanSession = (ScanSession) server.getSession(scanID);
    String oldThreadName = Thread.currentThread().getName();
    try {
        if (isCancelled() || scanSession == null)
            return;
        runState.set(ScanRunState.RUNNING);
        Thread.currentThread().setName("User: " + scanSession.getUser() + " Start: " + scanSession.startTime + " Client: " + scanSession.client + " Tablet: " + scanSession.extent);
        Tablet tablet = server.getOnlineTablet(scanSession.extent);
        if (tablet == null) {
            addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
            return;
        }
        long t1 = System.currentTimeMillis();
        ScanBatch batch = scanSession.scanner.read();
        long t2 = System.currentTimeMillis();
        scanSession.nbTimes.addStat(t2 - t1);
        // there should only be one thing on the queue at a time, so
        // it should be ok to call add()
        // instead of put()... if add() fails because queue is at
        // capacity it means there is code
        // problem somewhere
        addResult(batch);
    } catch (TabletClosedException e) {
        addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
    } catch (IterationInterruptedException iie) {
        if (!isCancelled()) {
            log.warn("Iteration interrupted, when scan not cancelled", iie);
            addResult(iie);
        }
    } catch (TooManyFilesException | SampleNotPresentException e) {
        addResult(e);
    } catch (OutOfMemoryError ome) {
        Halt.halt("Ran out of memory scanning " + scanSession.extent + " for " + scanSession.client, 1);
        addResult(ome);
    } catch (Throwable e) {
        log.warn("exception while scanning tablet " + (scanSession == null ? "(unknown)" : scanSession.extent), e);
        addResult(e);
    } finally {
        runState.set(ScanRunState.FINISHED);
        Thread.currentThread().setName(oldThreadName);
    }
}