Search in sources :

Example 1 with LinkedOperationQueueFactory

use of net.spy.memcached.ops.LinkedOperationQueueFactory in project druid by druid-io.

the class MemcachedCache method create.

public static MemcachedCache create(final MemcachedCacheConfig config) {
    final ConcurrentMap<String, AtomicLong> counters = new ConcurrentHashMap<>();
    final ConcurrentMap<String, AtomicLong> meters = new ConcurrentHashMap<>();
    final AbstractMonitor monitor = new AbstractMonitor() {

        final AtomicReference<Map<String, Long>> priorValues = new AtomicReference<Map<String, Long>>(new HashMap<String, Long>());

        @Override
        public boolean doMonitor(ServiceEmitter emitter) {
            final Map<String, Long> priorValues = this.priorValues.get();
            final Map<String, Long> currentValues = getCurrentValues();
            final ServiceMetricEvent.Builder builder = ServiceMetricEvent.builder();
            for (Map.Entry<String, Long> entry : currentValues.entrySet()) {
                emitter.emit(builder.setDimension("memcached metric", entry.getKey()).build("query/cache/memcached/total", entry.getValue()));
                final Long prior = priorValues.get(entry.getKey());
                if (prior != null) {
                    emitter.emit(builder.setDimension("memcached metric", entry.getKey()).build("query/cache/memcached/delta", entry.getValue() - prior));
                }
            }
            if (!this.priorValues.compareAndSet(priorValues, currentValues)) {
                log.error("Prior value changed while I was reporting! updating anyways");
                this.priorValues.set(currentValues);
            }
            return true;
        }

        private Map<String, Long> getCurrentValues() {
            final ImmutableMap.Builder<String, Long> builder = ImmutableMap.builder();
            for (Map.Entry<String, AtomicLong> entry : counters.entrySet()) {
                builder.put(entry.getKey(), entry.getValue().get());
            }
            for (Map.Entry<String, AtomicLong> entry : meters.entrySet()) {
                builder.put(entry.getKey(), entry.getValue().get());
            }
            return builder.build();
        }
    };
    try {
        LZ4Transcoder transcoder = new LZ4Transcoder(config.getMaxObjectSize());
        // always use compression
        transcoder.setCompressionThreshold(0);
        OperationQueueFactory opQueueFactory;
        long maxQueueBytes = config.getMaxOperationQueueSize();
        if (maxQueueBytes > 0) {
            opQueueFactory = new MemcachedOperationQueueFactory(maxQueueBytes);
        } else {
            opQueueFactory = new LinkedOperationQueueFactory();
        }
        final Predicate<String> interesting = new Predicate<String>() {

            // See net.spy.memcached.MemcachedConnection.registerMetrics()
            private final Set<String> interestingMetrics = ImmutableSet.of("[MEM] Reconnecting Nodes (ReconnectQueue)", //"[MEM] Shutting Down Nodes (NodesToShutdown)", // Busted
            "[MEM] Request Rate: All", "[MEM] Average Bytes written to OS per write", "[MEM] Average Bytes read from OS per read", "[MEM] Average Time on wire for operations (µs)", "[MEM] Response Rate: All (Failure + Success + Retry)", "[MEM] Response Rate: Retry", "[MEM] Response Rate: Failure", "[MEM] Response Rate: Success");

            @Override
            public boolean apply(@Nullable String input) {
                return input != null && interestingMetrics.contains(input);
            }
        };
        final MetricCollector metricCollector = new MetricCollector() {

            @Override
            public void addCounter(String name) {
                if (!interesting.apply(name)) {
                    return;
                }
                counters.putIfAbsent(name, new AtomicLong(0L));
                if (log.isDebugEnabled()) {
                    log.debug("Add Counter [%s]", name);
                }
            }

            @Override
            public void removeCounter(String name) {
                if (log.isDebugEnabled()) {
                    log.debug("Ignoring request to remove [%s]", name);
                }
            }

            @Override
            public void incrementCounter(String name) {
                if (!interesting.apply(name)) {
                    return;
                }
                AtomicLong counter = counters.get(name);
                if (counter == null) {
                    counters.putIfAbsent(name, new AtomicLong(0));
                    counter = counters.get(name);
                }
                counter.incrementAndGet();
                if (log.isDebugEnabled()) {
                    log.debug("Increment [%s]", name);
                }
            }

            @Override
            public void incrementCounter(String name, int amount) {
                if (!interesting.apply(name)) {
                    return;
                }
                AtomicLong counter = counters.get(name);
                if (counter == null) {
                    counters.putIfAbsent(name, new AtomicLong(0));
                    counter = counters.get(name);
                }
                counter.addAndGet(amount);
                if (log.isDebugEnabled()) {
                    log.debug("Increment [%s] %d", name, amount);
                }
            }

            @Override
            public void decrementCounter(String name) {
                if (!interesting.apply(name)) {
                    return;
                }
                AtomicLong counter = counters.get(name);
                if (counter == null) {
                    counters.putIfAbsent(name, new AtomicLong(0));
                    counter = counters.get(name);
                }
                counter.decrementAndGet();
                if (log.isDebugEnabled()) {
                    log.debug("Decrement [%s]", name);
                }
            }

            @Override
            public void decrementCounter(String name, int amount) {
                if (!interesting.apply(name)) {
                    return;
                }
                AtomicLong counter = counters.get(name);
                if (counter == null) {
                    counters.putIfAbsent(name, new AtomicLong(0L));
                    counter = counters.get(name);
                }
                counter.addAndGet(-amount);
                if (log.isDebugEnabled()) {
                    log.debug("Decrement [%s] %d", name, amount);
                }
            }

            @Override
            public void addMeter(String name) {
                if (!interesting.apply(name)) {
                    return;
                }
                meters.putIfAbsent(name, new AtomicLong(0L));
                if (log.isDebugEnabled()) {
                    log.debug("Adding meter [%s]", name);
                }
            }

            @Override
            public void removeMeter(String name) {
                if (!interesting.apply(name)) {
                    return;
                }
                if (log.isDebugEnabled()) {
                    log.debug("Ignoring request to remove meter [%s]", name);
                }
            }

            @Override
            public void markMeter(String name) {
                if (!interesting.apply(name)) {
                    return;
                }
                AtomicLong meter = meters.get(name);
                if (meter == null) {
                    meters.putIfAbsent(name, new AtomicLong(0L));
                    meter = meters.get(name);
                }
                meter.incrementAndGet();
                if (log.isDebugEnabled()) {
                    log.debug("Increment counter [%s]", name);
                }
            }

            @Override
            public void addHistogram(String name) {
                log.debug("Ignoring add histogram [%s]", name);
            }

            @Override
            public void removeHistogram(String name) {
                log.debug("Ignoring remove histogram [%s]", name);
            }

            @Override
            public void updateHistogram(String name, int amount) {
                log.debug("Ignoring update histogram [%s]: %d", name, amount);
            }
        };
        final ConnectionFactory connectionFactory = new MemcachedCustomConnectionFactoryBuilder().setKetamaNodeRepetitions(1000).setHashAlg(MURMUR3_128).setProtocol(ConnectionFactoryBuilder.Protocol.BINARY).setLocatorType(ConnectionFactoryBuilder.Locator.CONSISTENT).setDaemon(true).setFailureMode(FailureMode.Cancel).setTranscoder(transcoder).setShouldOptimize(true).setOpQueueMaxBlockTime(config.getTimeout()).setOpTimeout(config.getTimeout()).setReadBufferSize(config.getReadBufferSize()).setOpQueueFactory(opQueueFactory).setMetricCollector(metricCollector).setEnableMetrics(// Not as scary as it sounds
        MetricType.DEBUG).build();
        final List<InetSocketAddress> hosts = AddrUtil.getAddresses(config.getHosts());
        final Supplier<ResourceHolder<MemcachedClientIF>> clientSupplier;
        if (config.getNumConnections() > 1) {
            clientSupplier = new MemcacheClientPool(config.getNumConnections(), new Supplier<MemcachedClientIF>() {

                @Override
                public MemcachedClientIF get() {
                    try {
                        return new MemcachedClient(connectionFactory, hosts);
                    } catch (IOException e) {
                        log.error(e, "Unable to create memcached client");
                        throw Throwables.propagate(e);
                    }
                }
            });
        } else {
            clientSupplier = Suppliers.<ResourceHolder<MemcachedClientIF>>ofInstance(StupidResourceHolder.<MemcachedClientIF>create(new MemcachedClient(connectionFactory, hosts)));
        }
        return new MemcachedCache(clientSupplier, config, monitor);
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }
}
Also used : ServiceEmitter(com.metamx.emitter.service.ServiceEmitter) ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) InetSocketAddress(java.net.InetSocketAddress) Predicate(com.google.common.base.Predicate) ConnectionFactory(net.spy.memcached.ConnectionFactory) MemcachedClientIF(net.spy.memcached.MemcachedClientIF) MemcachedClient(net.spy.memcached.MemcachedClient) Supplier(com.google.common.base.Supplier) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ResourceHolder(io.druid.collections.ResourceHolder) StupidResourceHolder(io.druid.collections.StupidResourceHolder) AtomicReference(java.util.concurrent.atomic.AtomicReference) IOException(java.io.IOException) ImmutableMap(com.google.common.collect.ImmutableMap) AtomicLong(java.util.concurrent.atomic.AtomicLong) LinkedOperationQueueFactory(net.spy.memcached.ops.LinkedOperationQueueFactory) OperationQueueFactory(net.spy.memcached.ops.OperationQueueFactory) AbstractMonitor(com.metamx.metrics.AbstractMonitor) AtomicLong(java.util.concurrent.atomic.AtomicLong) MetricCollector(net.spy.memcached.metrics.MetricCollector) ServiceMetricEvent(com.metamx.emitter.service.ServiceMetricEvent) LinkedOperationQueueFactory(net.spy.memcached.ops.LinkedOperationQueueFactory) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ConcurrentMap(java.util.concurrent.ConcurrentMap) Nullable(javax.annotation.Nullable)

Aggregations

Predicate (com.google.common.base.Predicate)1 Supplier (com.google.common.base.Supplier)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 ServiceEmitter (com.metamx.emitter.service.ServiceEmitter)1 ServiceMetricEvent (com.metamx.emitter.service.ServiceMetricEvent)1 AbstractMonitor (com.metamx.metrics.AbstractMonitor)1 ResourceHolder (io.druid.collections.ResourceHolder)1 StupidResourceHolder (io.druid.collections.StupidResourceHolder)1 IOException (java.io.IOException)1 InetSocketAddress (java.net.InetSocketAddress)1 HashMap (java.util.HashMap)1 Map (java.util.Map)1 Set (java.util.Set)1 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)1 ConcurrentMap (java.util.concurrent.ConcurrentMap)1 AtomicLong (java.util.concurrent.atomic.AtomicLong)1 AtomicReference (java.util.concurrent.atomic.AtomicReference)1 Nullable (javax.annotation.Nullable)1 ConnectionFactory (net.spy.memcached.ConnectionFactory)1