Examples with CloseableIterator io.cdap.cdap.api.dataset.lib.CloseableIterator used on opensource projects

Example 1 with CloseableIterator

use of io.cdap.cdap.api.dataset.lib.CloseableIterator in project cdap by caskdata.

the class MockLogReader method getLog.

@Override
public CloseableIterator<LogEvent> getLog(LoggingContext loggingContext, long fromTimeMs, long toTimeMs, Filter filter) {
    CollectingCallback collectingCallback = new CollectingCallback();
    // since its just for test case, we don't need to lazily read logs (which is the purpose of returning an Iterator)
    long fromOffset = getOffset(fromTimeMs / 1000);
    long toOffset = getOffset(toTimeMs / 1000);
    getLogNext(loggingContext, new ReadRange(fromTimeMs, toTimeMs, fromOffset), (int) (toOffset - fromOffset), filter, collectingCallback);
    final Iterator<LogEvent> iterator = collectingCallback.getLogEvents().iterator();
    return new CloseableIterator<LogEvent>() {

        @Override
        public boolean hasNext() {
            return iterator.hasNext();
        }

        @Override
        public LogEvent next() {
            return iterator.next();
        }

        @Override
        public void remove() {
            iterator.remove();
        }

        @Override
        public void close() {
        // no-op
        }
    };
}

Example 2 with CloseableIterator

use of io.cdap.cdap.api.dataset.lib.CloseableIterator in project cdap by caskdata.

the class ObjectStoreDatasetTest method testScanObjectStore.

@Test
public void testScanObjectStore() throws Exception {
    DatasetId scan = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("scan");
    createObjectStoreInstance(scan, String.class);
    final ObjectStoreDataset<String> t = dsFrameworkUtil.getInstance(scan);
    TransactionExecutor txnl = dsFrameworkUtil.newTransactionExecutor(t);
    // write 10 values
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            for (int i = 0; i < 10; i++) {
                byte[] key = Bytes.toBytes(i);
                t.write(key, String.valueOf(i));
            }
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Iterator<KeyValue<byte[], String>> objectsIterator = t.scan(Bytes.toBytes(0), Bytes.toBytes(10));
            int sum = 0;
            while (objectsIterator.hasNext()) {
                sum += Integer.parseInt(objectsIterator.next().getValue());
            }
            // checking the sum equals sum of values from (0..9) which are the rows written and scanned for.
            Assert.assertEquals(45, sum);
        }
    });
    // start a transaction, scan part of them elements using scanner, close the scanner,
    // then call next() on scanner, it should fail
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            CloseableIterator<KeyValue<byte[], String>> objectsIterator = t.scan(Bytes.toBytes(0), Bytes.toBytes(10));
            int rowCount = 0;
            while (objectsIterator.hasNext() && (rowCount < 5)) {
                rowCount++;
            }
            objectsIterator.close();
            try {
                objectsIterator.next();
                Assert.fail("Reading after closing Scanner returned result.");
            } catch (NoSuchElementException e) {
            }
        }
    });
    dsFrameworkUtil.deleteInstance(scan);
}

Example 3 with CloseableIterator

use of io.cdap.cdap.api.dataset.lib.CloseableIterator in project cdap by caskdata.

the class FileLogReader method concat.

/**
 * See {@link com.google.common.collect.Iterators#concat(Iterator)}. The difference is that the input types and return
 * type are CloseableIterator, which closes the inputs that it has opened.
 */
public static <T> CloseableIterator<T> concat(final CloseableIterator<? extends CloseableIterator<? extends T>> inputs) {
    Preconditions.checkNotNull(inputs);
    return new CloseableIterator<T>() {

        @Override
        public void close() {
            current.close();
            current = null;
            while (inputs.hasNext()) {
                inputs.next().close();
            }
            inputs.close();
            removeFrom = null;
        }

        CloseableIterator<? extends T> current = new CloseableIterator<T>() {

            @Override
            public void close() {
            }

            @Override
            public boolean hasNext() {
                return false;
            }

            @Override
            public T next() {
                throw new NoSuchElementException();
            }

            @Override
            public void remove() {
                throw new IllegalStateException();
            }
        };

        CloseableIterator<? extends T> removeFrom;

        @Override
        public boolean hasNext() {
            // http://code.google.com/p/google-collections/issues/detail?id=151
            // current.hasNext() might be relatively expensive, worth minimizing.
            boolean currentHasNext;
            // again, incorrectly moving beyond the error.
            while (!(currentHasNext = Preconditions.checkNotNull(current).hasNext()) && inputs.hasNext()) {
                current = inputs.next();
            }
            return currentHasNext;
        }

        @Override
        public T next() {
            if (!hasNext()) {
                throw new NoSuchElementException();
            }
            removeFrom = current;
            return current.next();
        }

        @Override
        public void remove() {
            Preconditions.checkState(removeFrom != null, "no calls to next() since last call to remove()");
            removeFrom.remove();
            removeFrom = null;
        }
    };
}

Example 4 with CloseableIterator

use of io.cdap.cdap.api.dataset.lib.CloseableIterator in project cdap by caskdata.

the class MessagingAppTestRun method testSparkMessaging.

@Test
public void testSparkMessaging() throws Exception {
    ApplicationManager appManager = deployWithArtifact(NAMESPACE, MessagingApp.class, artifactJar);
    final SparkManager sparkManager = appManager.getSparkManager(MessagingSpark.class.getSimpleName()).start();
    final MessageFetcher fetcher = getMessagingContext().getMessageFetcher();
    final AtomicReference<String> messageId = new AtomicReference<>();
    // Wait for the Spark to create the topic
    final MessagingAdmin messagingAdmin = getMessagingAdmin(NAMESPACE.getNamespace());
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            try {
                messagingAdmin.getTopicProperties(MessagingApp.TOPIC);
                return true;
            } catch (TopicNotFoundException e) {
                return false;
            }
        }
    }, 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // This is to verify failed transaction is not publishing anything.
    for (String expected : Arrays.asList("start", "block")) {
        Tasks.waitFor(expected, new Callable<String>() {

            @Override
            public String call() throws Exception {
                try (CloseableIterator<Message> iterator = fetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, 1, messageId.get())) {
                    if (!iterator.hasNext()) {
                        return null;
                    }
                    Message message = iterator.next();
                    messageId.set(message.getId());
                    return message.getPayloadAsString();
                }
            }
        }, 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    }
    // Publish a control message to unblock the Spark execution
    getMessagingContext().getMessagePublisher().publish(NAMESPACE.getNamespace(), MessagingApp.CONTROL_TOPIC, "go");
    // Expects a result message as "result-15", where 15 is the sum of 1,2,3,4,5
    Tasks.waitFor("result-15", new Callable<String>() {

        @Override
        public String call() throws Exception {
            try (CloseableIterator<Message> iterator = fetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, 1, messageId.get())) {
                if (!iterator.hasNext()) {
                    return null;
                }
                Message message = iterator.next();
                messageId.set(message.getId());
                return message.getPayloadAsString();
            }
        }
    }, 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 60, TimeUnit.SECONDS);
}

Example 5 with CloseableIterator

use of io.cdap.cdap.api.dataset.lib.CloseableIterator in project cdap by caskdata.

the class SparkTest method testTransaction.

@Test
public void testTransaction() throws Exception {
    ApplicationManager applicationManager = deploy(TestSparkApp.class);
    // Write some data to a local file
    File inputFile = TEMP_FOLDER.newFile();
    try (PrintWriter writer = new PrintWriter(Files.newBufferedWriter(inputFile.toPath(), StandardCharsets.UTF_8))) {
        writer.println("red fox");
        writer.println("brown fox");
        writer.println("grey fox");
        writer.println("brown bear");
        writer.println("black bear");
    }
    // Run the spark program
    SparkManager sparkManager = applicationManager.getSparkManager(TransactionSpark.class.getSimpleName());
    sparkManager.start(ImmutableMap.of("input.file", inputFile.getAbsolutePath(), "keyvalue.table", "KeyValueTable", "result.all.dataset", "SparkResult", "result.threshold", "2", "result.threshold.dataset", "SparkThresholdResult"));
    // Verify result from dataset before the Spark program terminates
    final DataSetManager<KeyValueTable> resultManager = getDataset("SparkThresholdResult");
    final KeyValueTable resultTable = resultManager.get();
    // Expect the threshold result dataset, with threshold >=2, contains [brown, fox, bear]
    Tasks.waitFor(ImmutableSet.of("brown", "fox", "bear"), () -> {
        // This is to start a new TX
        resultManager.flush();
        LOG.info("Reading from threshold result");
        try (CloseableIterator<KeyValue<byte[], byte[]>> itor = resultTable.scan(null, null)) {
            return ImmutableSet.copyOf(Iterators.transform(itor, input -> {
                String word = Bytes.toString(input.getKey());
                LOG.info("{}, {}", word, Bytes.toInt(input.getValue()));
                return word;
            }));
        }
    }, 3, TimeUnit.MINUTES, 1, TimeUnit.SECONDS);
    sparkManager.stop();
    sparkManager.waitForRun(ProgramRunStatus.KILLED, 60, TimeUnit.SECONDS);
}