Examples with Stopwatch org.apache.drill.shaded.guava.com.google.common.base.Stopwatch used on opensource projects

Example 51 with Stopwatch

use of org.apache.drill.shaded.guava.com.google.common.base.Stopwatch in project drill by apache.

the class MessageIterator method hasNext.

@Override
public boolean hasNext() {
    if (recordIter != null && recordIter.hasNext()) {
        return true;
    }
    long nextPosition = kafkaConsumer.position(topicPartition);
    if (nextPosition >= endOffset) {
        return false;
    }
    ConsumerRecords<byte[], byte[]> consumerRecords;
    Stopwatch stopwatch = logger.isDebugEnabled() ? Stopwatch.createStarted() : null;
    try {
        consumerRecords = kafkaConsumer.poll(Duration.ofMillis(kafkaPollTimeOut));
    } catch (KafkaException ke) {
        throw UserException.dataReadError(ke).message(ke.getMessage()).build(logger);
    } finally {
        if (stopwatch != null) {
            stopwatch.stop();
        }
    }
    if (consumerRecords.isEmpty()) {
        throw UserException.dataReadError().message("Failed to fetch messages within %s milliseconds. " + "Consider increasing the value of the property: %s", kafkaPollTimeOut, ExecConstants.KAFKA_POLL_TIMEOUT).build(logger);
    }
    if (stopwatch != null) {
        long lastFetchTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);
        logger.debug("Time taken to fetch : {} milliseconds", lastFetchTime);
        totalFetchTime += lastFetchTime;
        logger.debug("Total number of messages fetched : {}", consumerRecords.count());
    }
    recordIter = consumerRecords.iterator();
    return recordIter.hasNext();
}

Example 52 with Stopwatch

use of org.apache.drill.shaded.guava.com.google.common.base.Stopwatch in project drill by apache.

the class SortTemplate method sort.

@Override
public void sort(SelectionVector4 vector4, VectorContainer container) {
    Stopwatch watch = Stopwatch.createStarted();
    QuickSort qs = new QuickSort();
    qs.sort(this, 0, vector4.getTotalCount());
    logger.debug("Took {} us to sort {} records", watch.elapsed(TimeUnit.MICROSECONDS), vector4.getTotalCount());
}

Example 53 with Stopwatch

use of org.apache.drill.shaded.guava.com.google.common.base.Stopwatch in project drill by apache.

the class Drillbit method close.

/**
 * The drillbit is moved into Quiescent state and the drillbit waits for grace
 * period amount of time. Then drillbit moves into draining state and waits
 * for all the queries and fragments to complete.
 */
@Override
public synchronized void close() {
    if (!stateManager.getState().equals(DrillbitState.ONLINE)) {
        return;
    }
    final Stopwatch w = Stopwatch.createStarted();
    logger.debug("Shutdown begun.");
    // We don't really want for Drillbits to pile up in memory, so the hook should be removed
    // It might be better to use PhantomReferences to cleanup as soon as Drillbit becomes
    // unreachable, however current approach seems to be good enough.
    Thread shutdownHook = this.shutdownHook;
    if (shutdownHook != null && Thread.currentThread() != shutdownHook) {
        try {
            Runtime.getRuntime().removeShutdownHook(shutdownHook);
        } catch (IllegalArgumentException e) {
        // If shutdown is in progress, just ignore the removal
        }
    }
    updateState(State.QUIESCENT);
    stateManager.setState(DrillbitState.GRACE);
    waitForGracePeriod();
    stateManager.setState(DrillbitState.DRAINING);
    // wait for all the in-flight queries to finish
    manager.waitToExit(forcefulShutdown);
    // safe to exit
    updateState(State.OFFLINE);
    stateManager.setState(DrillbitState.OFFLINE);
    if (quiescentMode) {
        return;
    }
    if (coord != null && registrationHandle != null) {
        coord.unregister(registrationHandle);
    }
    try {
        Thread.sleep(context.getConfig().getInt(ExecConstants.ZK_REFRESH) * 2);
    } catch (final InterruptedException e) {
        logger.warn("Interrupted while sleeping during coordination deregistration.");
        // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the
        // interruption and respond to it if it wants to.
        Thread.currentThread().interrupt();
    }
    try {
        AutoCloseables.close(webServer, engine, storeProvider, coord, manager, storageRegistry, context);
        // Closing the profile store provider if distinct
        if (storeProvider != profileStoreProvider) {
            AutoCloseables.close(profileStoreProvider);
        }
    } catch (Exception e) {
        logger.warn("Failure on close()", e);
    }
    logger.info("Shutdown completed ({} ms).", w.elapsed(TimeUnit.MILLISECONDS));
    stateManager.setState(DrillbitState.SHUTDOWN);
    // Interrupt GracefulShutdownThread since Drillbit close is not called from it.
    if (interruptPollShutdown) {
        gracefulShutdownThread.interrupt();
    }
}

Example 54 with Stopwatch

use of org.apache.drill.shaded.guava.com.google.common.base.Stopwatch in project drill by apache.

the class DrillConfig method create.

/**
 * @param overrideFileResourcePathname
 *          see {@link #create(String)}'s {@code overrideFileResourcePathname}
 * @param overriderProps
 *          optional property map for further overriding (after override file
 *          is assimilated
 * @param enableServerConfigs
 *          whether to enable server-specific configuration options
 * @param configInfo
 *          see {@link ConfigFileInfo}
 * @param fallbackConfig
 *          existing config which will be used as fallback
 * @return {@link DrillConfig} object with all configs from passed in resource files
 */
public static DrillConfig create(String overrideFileResourcePathname, Properties overriderProps, boolean enableServerConfigs, ConfigFileInfo configInfo, ConfigMergeable fallbackConfig) {
    final StringBuilder logString = new StringBuilder();
    final Stopwatch watch = Stopwatch.createStarted();
    overrideFileResourcePathname = overrideFileResourcePathname == null ? configInfo.getOverrideFileName() : overrideFileResourcePathname;
    // 1. Load defaults configuration file.
    Config fallback = ConfigFactory.empty();
    final ClassLoader[] classLoaders = ClasspathHelper.classLoaders();
    for (ClassLoader classLoader : classLoaders) {
        final URL url = classLoader.getResource(configInfo.getDefaultFileName());
        if (null != url) {
            logString.append("Base Configuration:\n\t- ").append(url).append("\n");
            fallback = ConfigFactory.load(classLoader, configInfo.getDefaultFileName());
            break;
        }
    }
    // 2. Load per-module configuration files.
    final String perModuleResourcePathName = configInfo.getModuleFileName();
    final Collection<URL> urls = (perModuleResourcePathName != null) ? ClassPathScanner.getConfigURLs(perModuleResourcePathName) : new ArrayList<>();
    logString.append("\nIntermediate Configuration and Plugin files, in order of precedence:\n");
    for (URL url : urls) {
        logString.append("\t- ").append(url).append("\n");
        fallback = ConfigFactory.parseURL(url).withFallback(fallback);
    }
    logString.append("\n");
    // Add fallback config for default and module configuration
    if (fallbackConfig != null) {
        fallback = fallback.withFallback(fallbackConfig);
    }
    final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
    // 3. Load distribution specific configuration file.
    final URL distribConfigFileUrl = classLoader.getResource(configInfo.getDistributionFileName());
    if (distribConfigFileUrl != null) {
        logString.append("Distribution Specific Configuration File: ").append(distribConfigFileUrl).append("\n");
    }
    fallback = ConfigFactory.load(configInfo.getDistributionFileName()).withFallback(fallback);
    // 4. Load any specified overrides configuration file along with any
    // overrides from JVM system properties (e.g., {-Dname=value").
    // (Per ConfigFactory.load(...)'s mention of using Thread.getContextClassLoader():)
    final URL overrideFileUrl = classLoader.getResource(overrideFileResourcePathname);
    if (overrideFileUrl != null) {
        logString.append("Override File: ").append(overrideFileUrl).append("\n");
    }
    Config effectiveConfig = ConfigFactory.load(overrideFileResourcePathname).withFallback(fallback);
    // 5. Apply any overriding properties.
    if (overriderProps != null) {
        logString.append("Overridden Properties:\n");
        for (Entry<Object, Object> entry : overriderProps.entrySet()) {
            if (!entry.getKey().equals("password")) {
                logString.append("\t-").append(entry.getKey()).append(" = ").append(entry.getValue()).append("\n");
            }
        }
        logString.append("\n");
        effectiveConfig = ConfigFactory.parseProperties(overriderProps).withFallback(effectiveConfig);
    }
    // 6. Create DrillConfig object from Config object.
    logger.info("Configuration and plugin file(s) identified in {}ms.\n{}", watch.elapsed(TimeUnit.MILLISECONDS), logString);
    return new DrillConfig(effectiveConfig.resolve());
}

Example 55 with Stopwatch

use of org.apache.drill.shaded.guava.com.google.common.base.Stopwatch in project drill by apache.

the class HBaseRecordReader method next.

@Override
public int next() {
    Stopwatch watch = Stopwatch.createStarted();
    if (rowKeyVector != null) {
        rowKeyVector.clear();
        rowKeyVector.allocateNew();
    }
    for (ValueVector v : familyVectorMap.values()) {
        v.clear();
        v.allocateNew();
    }
    int rowCount = 0;
    // if allocated memory for the first row is larger than allowed max in batch, it will be added anyway
    do {
        Result result = null;
        final OperatorStats operatorStats = operatorContext == null ? null : operatorContext.getStats();
        try {
            if (operatorStats != null) {
                operatorStats.startWait();
            }
            try {
                result = resultScanner.next();
            } finally {
                if (operatorStats != null) {
                    operatorStats.stopWait();
                }
            }
        } catch (IOException e) {
            throw new DrillRuntimeException(e);
        }
        if (result == null) {
            break;
        }
        // parse the result and populate the value vectors
        Cell[] cells = result.rawCells();
        if (rowKeyVector != null) {
            rowKeyVector.getMutator().setSafe(rowCount, cells[0].getRowArray(), cells[0].getRowOffset(), cells[0].getRowLength());
        }
        if (!rowKeyOnly) {
            for (final Cell cell : cells) {
                final int familyOffset = cell.getFamilyOffset();
                final int familyLength = cell.getFamilyLength();
                final byte[] familyArray = cell.getFamilyArray();
                final MapVector mv = getOrCreateFamilyVector(new String(familyArray, familyOffset, familyLength), true);
                final int qualifierOffset = cell.getQualifierOffset();
                final int qualifierLength = cell.getQualifierLength();
                final byte[] qualifierArray = cell.getQualifierArray();
                final NullableVarBinaryVector v = getOrCreateColumnVector(mv, new String(qualifierArray, qualifierOffset, qualifierLength));
                final int valueOffset = cell.getValueOffset();
                final int valueLength = cell.getValueLength();
                final byte[] valueArray = cell.getValueArray();
                v.getMutator().setSafe(rowCount, valueArray, valueOffset, valueLength);
            }
        }
        rowCount++;
    } while (canAddNewRow(rowCount));
    setOutputRowCount(rowCount);
    logger.debug("Took {} ms to get {} records", watch.elapsed(TimeUnit.MILLISECONDS), rowCount);
    return rowCount;
}