Examples with AbstractIterator com.google.common.collect.AbstractIterator used on opensource projects

Example 11 with AbstractIterator

use of com.google.common.collect.AbstractIterator in project atlasdb by palantir.

the class SnapshotTransaction method partitionByRow.

/**
 * Partitions a {@link RowColumnRangeIterator} into contiguous blocks that share the same row name.
 * {@link KeyValueService#getRowsColumnRange(TableReference, Iterable, ColumnRangeSelection, int, long)} guarantees
 * that all columns for a single row are adjacent, so this method will return an {@link Iterator} with exactly one
 * entry per non-empty row.
 */
private Iterator<Map.Entry<byte[], RowColumnRangeIterator>> partitionByRow(RowColumnRangeIterator rawResults) {
    PeekingIterator<Map.Entry<Cell, Value>> peekableRawResults = Iterators.peekingIterator(rawResults);
    return new AbstractIterator<Map.Entry<byte[], RowColumnRangeIterator>>() {

        byte[] prevRowName;

        @Override
        protected Map.Entry<byte[], RowColumnRangeIterator> computeNext() {
            finishConsumingPreviousRow(peekableRawResults);
            if (!peekableRawResults.hasNext()) {
                return endOfData();
            }
            byte[] nextRowName = peekableRawResults.peek().getKey().getRowName();
            Iterator<Map.Entry<Cell, Value>> columnsIterator = new AbstractIterator<Map.Entry<Cell, Value>>() {

                @Override
                protected Map.Entry<Cell, Value> computeNext() {
                    if (!peekableRawResults.hasNext() || !Arrays.equals(peekableRawResults.peek().getKey().getRowName(), nextRowName)) {
                        return endOfData();
                    }
                    return peekableRawResults.next();
                }
            };
            prevRowName = nextRowName;
            return Maps.immutableEntry(nextRowName, new LocalRowColumnRangeIterator(columnsIterator));
        }

        private void finishConsumingPreviousRow(PeekingIterator<Map.Entry<Cell, Value>> iter) {
            int numConsumed = 0;
            while (iter.hasNext() && Arrays.equals(iter.peek().getKey().getRowName(), prevRowName)) {
                iter.next();
                numConsumed++;
            }
            if (numConsumed > 0) {
                log.warn("Not all columns for row {} were read. {} columns were discarded.", UnsafeArg.of("row", Arrays.toString(prevRowName)), SafeArg.of("numColumnsDiscarded", numConsumed));
            }
        }
    };
}

Example 12 with AbstractIterator

use of com.google.common.collect.AbstractIterator in project cdap by caskdata.

the class MessageTableTest method testConcurrentWrites.

@Test
public void testConcurrentWrites() throws Exception {
    // Create two threads, each of them writes to a different topic with two events in one store call.
    // The iterators in the two threads would alternate to produce payload. This is for testing CDAP-12013
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch storeCompletion = new CountDownLatch(2);
    for (int i = 0; i < 2; i++) {
        final TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
        TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
        try (MetadataTable metadataTable = getMetadataTable()) {
            metadataTable.createTopic(metadata);
        }
        final int threadId = i;
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                try (MessageTable messageTable = getMessageTable()) {
                    messageTable.store(new AbstractIterator<MessageTable.Entry>() {

                        int messageCount = 0;

                        @Override
                        protected MessageTable.Entry computeNext() {
                            if (messageCount >= 2) {
                                return endOfData();
                            }
                            try {
                                barrier.await();
                            } catch (Exception e) {
                                throw Throwables.propagate(e);
                            }
                            return new TestMessageEntry(topicId, GENERATION, System.currentTimeMillis(), messageCount, null, Bytes.toBytes("message " + threadId + " " + messageCount++));
                        }
                    });
                    storeCompletion.countDown();
                } catch (Exception e) {
                    LOG.error("Failed to store to MessageTable", e);
                }
                return null;
            }
        });
    }
    executor.shutdown();
    Assert.assertTrue(storeCompletion.await(30, TimeUnit.SECONDS));
    // Read from each topic. Each topic should have two messages
    for (int i = 0; i < 2; i++) {
        TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
        TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
        try (MessageTable messageTable = getMessageTable();
            CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(metadata, 0, 10, null)) {
            List<MessageTable.Entry> entries = Lists.newArrayList(iterator);
            Assert.assertEquals(2, entries.size());
            int count = 0;
            for (MessageTable.Entry entry : entries) {
                Assert.assertEquals("message " + i + " " + count++, Bytes.toString(entry.getPayload()));
            }
        }
    }
}

Example 13 with AbstractIterator

use of com.google.common.collect.AbstractIterator in project cdap by caskdata.

the class PayloadTableTest method testConcurrentWrites.

@Test
public void testConcurrentWrites() throws Exception {
    // Create two threads, each of them writes to a different topic with two events in one store call.
    // The iterators in the two threads would alternate to produce payload. This is for testing CDAP-12013
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch storeCompletion = new CountDownLatch(2);
    for (int i = 0; i < 2; i++) {
        final TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
        TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
        try (MetadataTable metadataTable = getMetadataTable()) {
            metadataTable.createTopic(metadata);
        }
        final int threadId = i;
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                try (PayloadTable payloadTable = getPayloadTable()) {
                    payloadTable.store(new AbstractIterator<PayloadTable.Entry>() {

                        short messageCount = 0;

                        @Override
                        protected PayloadTable.Entry computeNext() {
                            if (messageCount >= 2) {
                                return endOfData();
                            }
                            try {
                                barrier.await();
                            } catch (Exception e) {
                                throw Throwables.propagate(e);
                            }
                            return new TestPayloadEntry(topicId, GENERATION, threadId, 0, messageCount, Bytes.toBytes("message " + threadId + " " + messageCount++));
                        }
                    });
                    storeCompletion.countDown();
                } catch (Exception e) {
                    LOG.error("Failed to store to MessageTable", e);
                }
                return null;
            }
        });
    }
    executor.shutdown();
    Assert.assertTrue(storeCompletion.await(5, TimeUnit.SECONDS));
    // Read from each topic. Each topic should have two messages
    for (int i = 0; i < 2; i++) {
        TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
        TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
        byte[] rawId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0, (short) 0, rawId, 0);
        MessageId messageId = new MessageId(rawId);
        try (PayloadTable payloadTable = getPayloadTable();
            CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(metadata, i, messageId, true, 10)) {
            List<PayloadTable.Entry> entries = Lists.newArrayList(iterator);
            Assert.assertEquals(2, entries.size());
            int count = 0;
            for (PayloadTable.Entry entry : entries) {
                Assert.assertEquals("message " + i + " " + count++, Bytes.toString(entry.getPayload()));
            }
        }
    }
}

Example 14 with AbstractIterator

use of com.google.common.collect.AbstractIterator in project cdap by caskdata.

the class MessageCacheTest method testAddError.

@Test
public void testAddError() throws Exception {
    // Test to verify various error situations are being safeguarded
    final MessageCache<Integer> cache = new MessageCache<>(new IntComparator(), new UnitWeigher<Integer>(), new MessageCache.Limits(5, 7, 10), NOOP_METRICS);
    // 1. Adding out of order should result in error
    try {
        cache.addAll(Arrays.asList(5, 2, 3, 4).iterator());
        Assert.fail("Expected failure for adding out of order");
    } catch (IllegalArgumentException e) {
        // Expected. The cache should be cleared
        Assert.assertEquals(0, cache.getCurrentWeight());
    }
    // 2. Adding entries that are smaller than the largest one in the cache
    cache.addAll(Arrays.asList(5, 6, 7, 8).iterator());
    try {
        cache.addAll(Arrays.asList(1, 2, 3, 4).iterator());
        Assert.fail("Expected failure for adding out of order");
    } catch (IllegalArgumentException e) {
        // Expected. The cache should be cleared
        Assert.assertEquals(0, cache.getCurrentWeight());
    }
    // 3. Adding duplicate entry
    try {
        cache.addAll(Arrays.asList(1, 1).iterator());
        Assert.fail("Expected failure for adding out of order");
    } catch (IllegalArgumentException e) {
        // Expected. The cache should be cleared
        Assert.assertEquals(0, cache.getCurrentWeight());
    }
    // 4. Adding entry that is already exist in the cache
    cache.addAll(Arrays.asList(1, 2).iterator());
    try {
        cache.addAll(Arrays.asList(2, 3).iterator());
    } catch (IllegalArgumentException e) {
        // Expected. The cache should be cleared
        Assert.assertEquals(0, cache.getCurrentWeight());
    }
    // 5. Multiple threads calling addAll at the same time
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch produceLatch = new CountDownLatch(1);
    // Starts the the first thread and block inside the addAll call
    new Thread() {

        @Override
        public void run() {
            cache.addAll(new AbstractIterator<Integer>() {

                private boolean produced;

                @Override
                protected Integer computeNext() {
                    try {
                        barrier.await();
                    } catch (Exception e) {
                    // This shouldn't happen
                    }
                    Uninterruptibles.awaitUninterruptibly(produceLatch);
                    if (!produced) {
                        produced = true;
                        return 1;
                    }
                    return endOfData();
                }
            });
        }
    }.start();
    // Starts the second thread that tries to call addAll after the first thread is blocked inside the addAll call
    final BlockingQueue<Exception> exception = new ArrayBlockingQueue<>(1);
    new Thread() {

        @Override
        public void run() {
            try {
                barrier.await();
            } catch (Exception e) {
            // This shouldn't happen
            }
            try {
                cache.addAll(Arrays.asList(1, 2, 3).iterator());
            } catch (Exception e) {
                exception.add(e);
            }
        }
    }.start();
    Exception e = exception.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(e);
    Assert.assertTrue(e instanceof ConcurrentModificationException);
    // Unblock the first thread
    produceLatch.countDown();
    final MessageFilter<Integer> filter = MessageFilter.alwaysAccept();
    Tasks.waitFor(Collections.singletonList(1), new Callable<List<Integer>>() {

        @Override
        public List<Integer> call() throws Exception {
            try (MessageCache.Scanner<Integer> scanner = cache.scan(0, true, 10, filter)) {
                return Lists.newArrayList(scanner);
            }
        }
    }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
}

Example 15 with AbstractIterator

use of com.google.common.collect.AbstractIterator in project cdap by caskdata.

the class AggregatedMetricsCollectionService method getMetrics.

private Iterator<MetricValues> getMetrics(final long timestamp) {
    // NOTE : emitters.asMap does not reset the access time in cache,
    // so it's the preferred way to access the cache entries. as we access and emit metrics every second.
    final Iterator<Map.Entry<Map<String, String>, LoadingCache<String, AggregatedMetricsEmitter>>> iterator = emitters.asMap().entrySet().iterator();
    return new AbstractIterator<MetricValues>() {

        @Override
        protected MetricValues computeNext() {
            while (iterator.hasNext()) {
                Map.Entry<Map<String, String>, LoadingCache<String, AggregatedMetricsEmitter>> entry = iterator.next();
                Map<String, AggregatedMetricsEmitter> metricEmitters = entry.getValue().asMap();
                // +1 because we add extra metric about how many metric values did we emit in this context (see below)
                List<MetricValue> metricValues = Lists.newArrayListWithCapacity(metricEmitters.size() + 1);
                for (Map.Entry<String, AggregatedMetricsEmitter> emitterEntry : metricEmitters.entrySet()) {
                    MetricValue metricValue = emitterEntry.getValue().emit();
                    // skip increment by 0
                    if (metricValue.getType() == MetricType.COUNTER && metricValue.getValue() == 0) {
                        continue;
                    }
                    metricValues.add(metricValue);
                }
                if (metricValues.isEmpty()) {
                    // skip if there are no metric values to send
                    continue;
                }
                // number of emitted metrics
                metricValues.add(new MetricValue("metrics.emitted.count", MetricType.COUNTER, metricValues.size() + 1));
                LOG.trace("Emit metric {}", metricValues);
                return new MetricValues(entry.getKey(), timestamp, metricValues);
            }
            return endOfData();
        }
    };
}