Examples with LoggingRunnable org.apache.accumulo.fate.util.LoggingRunnable used on opensource projects

Example 1 with LoggingRunnable

use of org.apache.accumulo.fate.util.LoggingRunnable in project accumulo by apache.

the class BulkImporter method estimateSizes.

private Map<Path, List<AssignmentInfo>> estimateSizes(final AccumuloConfiguration acuConf, final Configuration conf, final VolumeManager vm, Map<Path, List<TabletLocation>> assignments, Collection<Path> paths, int numThreads) {
    long t1 = System.currentTimeMillis();
    final Map<Path, Long> mapFileSizes = new TreeMap<>();
    try {
        for (Path path : paths) {
            FileSystem fs = vm.getVolumeByPath(path).getFileSystem();
            mapFileSizes.put(path, fs.getContentSummary(path).getLength());
        }
    } catch (IOException e) {
        log.error("Failed to get map files in for {}: {}", paths, e.getMessage(), e);
        throw new RuntimeException(e);
    }
    final Map<Path, List<AssignmentInfo>> ais = Collections.synchronizedMap(new TreeMap<Path, List<AssignmentInfo>>());
    ExecutorService threadPool = Executors.newFixedThreadPool(numThreads, new NamingThreadFactory("estimateSizes"));
    for (final Entry<Path, List<TabletLocation>> entry : assignments.entrySet()) {
        if (entry.getValue().size() == 1) {
            TabletLocation tabletLocation = entry.getValue().get(0);
            // if the tablet completely contains the map file, there is no
            // need to estimate its
            // size
            ais.put(entry.getKey(), Collections.singletonList(new AssignmentInfo(tabletLocation.tablet_extent, mapFileSizes.get(entry.getKey()))));
            continue;
        }
        Runnable estimationTask = new Runnable() {

            @Override
            public void run() {
                Map<KeyExtent, Long> estimatedSizes = null;
                try {
                    estimatedSizes = FileUtil.estimateSizes(acuConf, entry.getKey(), mapFileSizes.get(entry.getKey()), extentsOf(entry.getValue()), conf, vm);
                } catch (IOException e) {
                    log.warn("Failed to estimate map file sizes {}", e.getMessage());
                }
                if (estimatedSizes == null) {
                    // estimation failed, do a simple estimation
                    estimatedSizes = new TreeMap<>();
                    long estSize = (long) (mapFileSizes.get(entry.getKey()) / (double) entry.getValue().size());
                    for (TabletLocation tl : entry.getValue()) estimatedSizes.put(tl.tablet_extent, estSize);
                }
                List<AssignmentInfo> assignmentInfoList = new ArrayList<>(estimatedSizes.size());
                for (Entry<KeyExtent, Long> entry2 : estimatedSizes.entrySet()) assignmentInfoList.add(new AssignmentInfo(entry2.getKey(), entry2.getValue()));
                ais.put(entry.getKey(), assignmentInfoList);
            }
        };
        threadPool.submit(new TraceRunnable(new LoggingRunnable(log, estimationTask)));
    }
    threadPool.shutdown();
    while (!threadPool.isTerminated()) {
        try {
            threadPool.awaitTermination(60, TimeUnit.SECONDS);
        } catch (InterruptedException e) {
            log.error("Encountered InterruptedException while waiting for the threadPool to terminate.", e);
            throw new RuntimeException(e);
        }
    }
    long t2 = System.currentTimeMillis();
    log.debug(String.format("Estimated map files sizes in %6.2f secs", (t2 - t1) / 1000.0));
    return ais;
}

Example 2 with LoggingRunnable

use of org.apache.accumulo.fate.util.LoggingRunnable in project accumulo by apache.

the class BulkImporter method importFiles.

public AssignmentStats importFiles(List<String> files, Path failureDir) throws IOException, AccumuloException, AccumuloSecurityException, ThriftTableOperationException {
    int numThreads = context.getConfiguration().getCount(Property.TSERV_BULK_PROCESS_THREADS);
    int numAssignThreads = context.getConfiguration().getCount(Property.TSERV_BULK_ASSIGNMENT_THREADS);
    timer = new StopWatch<>(Timers.class);
    timer.start(Timers.TOTAL);
    Configuration conf = CachedConfiguration.getInstance();
    VolumeManagerImpl.get(context.getConfiguration());
    final VolumeManager fs = VolumeManagerImpl.get(context.getConfiguration());
    Set<Path> paths = new HashSet<>();
    for (String file : files) {
        paths.add(new Path(file));
    }
    AssignmentStats assignmentStats = new AssignmentStats(paths.size());
    final Map<Path, List<KeyExtent>> completeFailures = Collections.synchronizedSortedMap(new TreeMap<Path, List<KeyExtent>>());
    ClientService.Client client = null;
    final TabletLocator locator = TabletLocator.getLocator(context, Table.ID.of(tableId));
    try {
        final Map<Path, List<TabletLocation>> assignments = Collections.synchronizedSortedMap(new TreeMap<Path, List<TabletLocation>>());
        timer.start(Timers.EXAMINE_MAP_FILES);
        ExecutorService threadPool = Executors.newFixedThreadPool(numThreads, new NamingThreadFactory("findOverlapping"));
        for (Path path : paths) {
            final Path mapFile = path;
            Runnable getAssignments = new Runnable() {

                @Override
                public void run() {
                    List<TabletLocation> tabletsToAssignMapFileTo = Collections.emptyList();
                    try {
                        tabletsToAssignMapFileTo = findOverlappingTablets(context, fs, locator, mapFile);
                    } catch (Exception ex) {
                        log.warn("Unable to find tablets that overlap file " + mapFile.toString(), ex);
                    }
                    log.debug("Map file {} found to overlap {} tablets", mapFile, tabletsToAssignMapFileTo.size());
                    if (tabletsToAssignMapFileTo.size() == 0) {
                        List<KeyExtent> empty = Collections.emptyList();
                        completeFailures.put(mapFile, empty);
                    } else
                        assignments.put(mapFile, tabletsToAssignMapFileTo);
                }
            };
            threadPool.submit(new TraceRunnable(new LoggingRunnable(log, getAssignments)));
        }
        threadPool.shutdown();
        while (!threadPool.isTerminated()) {
            try {
                threadPool.awaitTermination(60, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
        }
        timer.stop(Timers.EXAMINE_MAP_FILES);
        assignmentStats.attemptingAssignments(assignments);
        Map<Path, List<KeyExtent>> assignmentFailures = assignMapFiles(context, conf, fs, tableId, assignments, paths, numAssignThreads, numThreads);
        assignmentStats.assignmentsFailed(assignmentFailures);
        Map<Path, Integer> failureCount = new TreeMap<>();
        for (Entry<Path, List<KeyExtent>> entry : assignmentFailures.entrySet()) failureCount.put(entry.getKey(), 1);
        long sleepTime = 2 * 1000;
        while (assignmentFailures.size() > 0) {
            sleepTime = Math.min(sleepTime * 2, 60 * 1000);
            locator.invalidateCache();
            // assumption about assignment failures is that it caused by a split
            // happening or a missing location
            // 
            // for splits we need to find children key extents that cover the
            // same key range and are contiguous (no holes, no overlap)
            timer.start(Timers.SLEEP);
            sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS);
            timer.stop(Timers.SLEEP);
            log.debug("Trying to assign {} map files that previously failed on some key extents", assignmentFailures.size());
            assignments.clear();
            // assign to
            for (Entry<Path, List<KeyExtent>> entry : assignmentFailures.entrySet()) {
                Iterator<KeyExtent> keListIter = entry.getValue().iterator();
                List<TabletLocation> tabletsToAssignMapFileTo = new ArrayList<>();
                while (keListIter.hasNext()) {
                    KeyExtent ke = keListIter.next();
                    timer.start(Timers.QUERY_METADATA);
                    try {
                        tabletsToAssignMapFileTo.addAll(findOverlappingTablets(context, fs, locator, entry.getKey(), ke));
                        keListIter.remove();
                    } catch (Exception ex) {
                        log.warn("Exception finding overlapping tablets, will retry tablet " + ke, ex);
                    }
                    timer.stop(Timers.QUERY_METADATA);
                }
                if (tabletsToAssignMapFileTo.size() > 0)
                    assignments.put(entry.getKey(), tabletsToAssignMapFileTo);
            }
            assignmentStats.attemptingAssignments(assignments);
            Map<Path, List<KeyExtent>> assignmentFailures2 = assignMapFiles(context, conf, fs, tableId, assignments, paths, numAssignThreads, numThreads);
            assignmentStats.assignmentsFailed(assignmentFailures2);
            // merge assignmentFailures2 into assignmentFailures
            for (Entry<Path, List<KeyExtent>> entry : assignmentFailures2.entrySet()) {
                assignmentFailures.get(entry.getKey()).addAll(entry.getValue());
                Integer fc = failureCount.get(entry.getKey());
                if (fc == null)
                    fc = 0;
                failureCount.put(entry.getKey(), fc + 1);
            }
            // remove map files that have no more key extents to assign
            Iterator<Entry<Path, List<KeyExtent>>> afIter = assignmentFailures.entrySet().iterator();
            while (afIter.hasNext()) {
                Entry<Path, List<KeyExtent>> entry = afIter.next();
                if (entry.getValue().size() == 0)
                    afIter.remove();
            }
            Set<Entry<Path, Integer>> failureIter = failureCount.entrySet();
            for (Entry<Path, Integer> entry : failureIter) {
                int retries = context.getConfiguration().getCount(Property.TSERV_BULK_RETRY);
                if (entry.getValue() > retries && assignmentFailures.get(entry.getKey()) != null) {
                    log.error("Map file {} failed more than {} times, giving up.", entry.getKey(), retries);
                    completeFailures.put(entry.getKey(), assignmentFailures.get(entry.getKey()));
                    assignmentFailures.remove(entry.getKey());
                }
            }
        }
        assignmentStats.assignmentsAbandoned(completeFailures);
        Set<Path> failedFailures = processFailures(completeFailures);
        assignmentStats.unrecoveredMapFiles(failedFailures);
        timer.stop(Timers.TOTAL);
        printReport(paths);
        return assignmentStats;
    } finally {
        if (client != null) {
            ServerClient.close(client);
        }
    }
}

Example 3 with LoggingRunnable

use of org.apache.accumulo.fate.util.LoggingRunnable in project accumulo by apache.

the class TabletServer method config.

public void config(String hostname) {
    log.info("Tablet server starting on {}", hostname);
    majorCompactorThread = new Daemon(new LoggingRunnable(log, new MajorCompactor(getConfiguration())));
    majorCompactorThread.setName("Split/MajC initiator");
    majorCompactorThread.start();
    clientAddress = HostAndPort.fromParts(hostname, 0);
    try {
        AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig() {

            @Override
            public Map<String, String> getVfsContextClasspathProperties() {
                return getConfiguration().getAllPropertiesWithPrefix(Property.VFS_CONTEXT_CLASSPATH_PROPERTY);
            }
        });
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    // A task that cleans up unused classloader contexts
    Runnable contextCleaner = new Runnable() {

        @Override
        public void run() {
            Set<String> contextProperties = getServerConfigurationFactory().getSystemConfiguration().getAllPropertiesWithPrefix(Property.VFS_CONTEXT_CLASSPATH_PROPERTY).keySet();
            Set<String> configuredContexts = new HashSet<>();
            for (String prop : contextProperties) {
                configuredContexts.add(prop.substring(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.name().length()));
            }
            try {
                AccumuloVFSClassLoader.getContextManager().removeUnusedContexts(configuredContexts);
            } catch (IOException e) {
                log.warn("{}", e.getMessage(), e);
            }
        }
    };
    AccumuloConfiguration aconf = getConfiguration();
    SimpleTimer.getInstance(aconf).schedule(contextCleaner, 60000, 60000);
    FileSystemMonitor.start(aconf, Property.TSERV_MONITOR_FS);
    Runnable gcDebugTask = new Runnable() {

        @Override
        public void run() {
            gcLogger.logGCInfo(getConfiguration());
        }
    };
    SimpleTimer.getInstance(aconf).schedule(gcDebugTask, 0, TIME_BETWEEN_GC_CHECKS);
    Runnable constraintTask = new Runnable() {

        @Override
        public void run() {
            ArrayList<Tablet> tablets;
            synchronized (onlineTablets) {
                tablets = new ArrayList<>(onlineTablets.values());
            }
            for (Tablet tablet : tablets) {
                tablet.checkConstraints();
            }
        }
    };
    SimpleTimer.getInstance(aconf).schedule(constraintTask, 0, 1000);
}

Example 4 with LoggingRunnable

use of org.apache.accumulo.fate.util.LoggingRunnable in project accumulo by apache.

the class DfsLogger method open.

/**
 * Opens a Write-Ahead Log file and writes the necessary header information and OPEN entry to the file. The file is ready to be used for ingest if this method
 * returns successfully. If an exception is thrown from this method, it is the callers responsibility to ensure that {@link #close()} is called to prevent
 * leaking the file handle and/or syncing thread.
 *
 * @param address
 *          The address of the host using this WAL
 */
public synchronized void open(String address) throws IOException {
    String filename = UUID.randomUUID().toString();
    log.debug("Address is {}", address);
    String logger = Joiner.on("+").join(address.split(":"));
    log.debug("DfsLogger.open() begin");
    VolumeManager fs = conf.getFileSystem();
    VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(ChooserScope.LOGGER);
    logPath = fs.choose(chooserEnv, ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.WAL_DIR + Path.SEPARATOR + logger + Path.SEPARATOR + filename;
    metaReference = toString();
    LoggerOperation op = null;
    try {
        short replication = (short) conf.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION);
        if (replication == 0)
            replication = fs.getDefaultReplication(new Path(logPath));
        long blockSize = getWalBlockSize(conf.getConfiguration());
        if (conf.getConfiguration().getBoolean(Property.TSERV_WAL_SYNC))
            logFile = fs.createSyncable(new Path(logPath), 0, replication, blockSize);
        else
            logFile = fs.create(new Path(logPath), true, 0, replication, blockSize);
        sync = logFile.getClass().getMethod("hsync");
        flush = logFile.getClass().getMethod("hflush");
        // Initialize the crypto operations.
        org.apache.accumulo.core.security.crypto.CryptoModule cryptoModule = org.apache.accumulo.core.security.crypto.CryptoModuleFactory.getCryptoModule(conf.getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
        // Initialize the log file with a header and the crypto params used to set up this log file.
        logFile.write(LOG_FILE_HEADER_V3.getBytes(UTF_8));
        CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf.getConfiguration());
        // Immediately update to the correct cipher. Doing this here keeps the CryptoModule independent of the writers using it
        if (params.getAllOptions().get(Property.CRYPTO_WAL_CIPHER_SUITE.getKey()) != null && !params.getAllOptions().get(Property.CRYPTO_WAL_CIPHER_SUITE.getKey()).equals("")) {
            params.setCipherSuite(params.getAllOptions().get(Property.CRYPTO_WAL_CIPHER_SUITE.getKey()));
        }
        NoFlushOutputStream nfos = new NoFlushOutputStream(logFile);
        params.setPlaintextOutputStream(nfos);
        // In order to bootstrap the reading of this file later, we have to record the CryptoModule that was used to encipher it here,
        // so that that crypto module can re-read its own parameters.
        logFile.writeUTF(conf.getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
        params = cryptoModule.getEncryptingOutputStream(params);
        OutputStream encipheringOutputStream = params.getEncryptedOutputStream();
        // another data OutputStream.
        if (encipheringOutputStream == nfos) {
            log.debug("No enciphering, using raw output stream");
            encryptingLogFile = nfos;
        } else {
            log.debug("Enciphering found, wrapping in DataOutputStream");
            encryptingLogFile = new DataOutputStream(encipheringOutputStream);
        }
        LogFileKey key = new LogFileKey();
        key.event = OPEN;
        key.tserverSession = filename;
        key.filename = filename;
        op = logFileData(Collections.singletonList(new Pair<>(key, EMPTY)), Durability.SYNC);
    } catch (Exception ex) {
        if (logFile != null)
            logFile.close();
        logFile = null;
        encryptingLogFile = null;
        throw new IOException(ex);
    }
    syncThread = new Daemon(new LoggingRunnable(log, new LogSyncingTask()));
    syncThread.setName("Accumulo WALog thread " + toString());
    syncThread.start();
    op.await();
    log.debug("Got new write-ahead log: {}", this);
}

Example 5 with LoggingRunnable

use of org.apache.accumulo.fate.util.LoggingRunnable in project accumulo by apache.

the class ConditionalWriterImpl method queue.

private void queue(String location, TabletServerMutations<QCMutation> mutations) {
    ServerQueue serverQueue = getServerQueue(location);
    synchronized (serverQueue) {
        serverQueue.queue.add(mutations);
        // never execute more than one task per server
        if (!serverQueue.taskQueued) {
            threadPool.execute(new LoggingRunnable(log, Trace.wrap(new SendTask(location))));
            serverQueue.taskQueued = true;
        }
    }
}