Examples with CloseableIterator co.cask.cdap.api.dataset.lib.CloseableIterator used on opensource projects

Example 6 with CloseableIterator

use of co.cask.cdap.api.dataset.lib.CloseableIterator in project cdap by caskdata.

the class DistributedLogFrameworkTest method testFramework.

@Test
public void testFramework() throws Exception {
    DistributedLogFramework framework = injector.getInstance(DistributedLogFramework.class);
    CConfiguration cConf = injector.getInstance(CConfiguration.class);
    framework.startAndWait();
    // Send some logs to Kafka.
    LoggingContext context = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, "test");
    // Make sure all events get flushed in the same batch
    long eventTimeBase = System.currentTimeMillis() + cConf.getInt(Constants.Logging.PIPELINE_EVENT_DELAY_MS);
    final int msgCount = 50;
    for (int i = 0; i < msgCount; i++) {
        // Publish logs in descending timestamp order
        publishLog(cConf.get(Constants.Logging.KAFKA_TOPIC), context, ImmutableList.of(createLoggingEvent("co.cask.test." + i, Level.INFO, "Testing " + i, eventTimeBase - i)));
    }
    // Read the logs back. They should be sorted by timestamp order.
    final FileMetaDataReader metaDataReader = injector.getInstance(FileMetaDataReader.class);
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            List<LogLocation> locations = metaDataReader.listFiles(new LogPathIdentifier(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, "test"), 0, Long.MAX_VALUE);
            if (locations.size() != 1) {
                return false;
            }
            LogLocation location = locations.get(0);
            int i = 0;
            try {
                try (CloseableIterator<LogEvent> iter = location.readLog(Filter.EMPTY_FILTER, 0, Long.MAX_VALUE, msgCount)) {
                    while (iter.hasNext()) {
                        String expectedMsg = "Testing " + (msgCount - i - 1);
                        LogEvent event = iter.next();
                        if (!expectedMsg.equals(event.getLoggingEvent().getMessage())) {
                            return false;
                        }
                        i++;
                    }
                    return i == msgCount;
                }
            } catch (Exception e) {
                // and the time when actual content are flushed to the file
                return false;
            }
        }
    }, 10, TimeUnit.SECONDS, msgCount, TimeUnit.MILLISECONDS);
    framework.stopAndWait();
    // Check the checkpoint is persisted correctly. Since all messages are processed,
    // the checkpoint should be the same as the message count.
    Checkpoint checkpoint = injector.getInstance(CheckpointManagerFactory.class).create(cConf.get(Constants.Logging.KAFKA_TOPIC), Bytes.toBytes(100)).getCheckpoint(0);
    Assert.assertEquals(msgCount, checkpoint.getNextOffset());
}

Example 7 with CloseableIterator

use of co.cask.cdap.api.dataset.lib.CloseableIterator in project cdap by caskdata.

the class SparkTestRun method testTransaction.

@Test
public void testTransaction() throws Exception {
    ApplicationManager applicationManager = deploy(TestSparkApp.class);
    StreamManager streamManager = getStreamManager("SparkStream");
    // Write some sentences to the stream
    streamManager.send("red fox");
    streamManager.send("brown fox");
    streamManager.send("grey fox");
    streamManager.send("brown bear");
    streamManager.send("black bear");
    // Run the spark program
    SparkManager sparkManager = applicationManager.getSparkManager(TransactionSpark.class.getSimpleName());
    sparkManager.start(ImmutableMap.of("source.stream", "SparkStream", "keyvalue.table", "KeyValueTable", "result.all.dataset", "SparkResult", "result.threshold", "2", "result.threshold.dataset", "SparkThresholdResult"));
    // Verify result from dataset before the Spark program terminates
    final DataSetManager<KeyValueTable> resultManager = getDataset("SparkThresholdResult");
    final KeyValueTable resultTable = resultManager.get();
    // Expect the threshold result dataset, with threshold >=2, contains [brown, fox, bear]
    Tasks.waitFor(ImmutableSet.of("brown", "fox", "bear"), new Callable<Set<String>>() {

        @Override
        public Set<String> call() throws Exception {
            // This is to start a new TX
            resultManager.flush();
            LOG.info("Reading from threshold result");
            try (CloseableIterator<KeyValue<byte[], byte[]>> itor = resultTable.scan(null, null)) {
                return ImmutableSet.copyOf(Iterators.transform(itor, new Function<KeyValue<byte[], byte[]>, String>() {

                    @Override
                    public String apply(KeyValue<byte[], byte[]> input) {
                        String word = Bytes.toString(input.getKey());
                        LOG.info("{}, {}", word, Bytes.toInt(input.getValue()));
                        return word;
                    }
                }));
            }
        }
    }, 3, TimeUnit.MINUTES, 1, TimeUnit.SECONDS);
    sparkManager.stop();
    sparkManager.waitForRun(ProgramRunStatus.KILLED, 60, TimeUnit.SECONDS);
}

Example 8 with CloseableIterator

use of co.cask.cdap.api.dataset.lib.CloseableIterator in project cdap by caskdata.

the class MessagingAppTestRun method testWithWorker.

@Test
public void testWithWorker() throws Exception {
    ApplicationManager appManager = deployWithArtifact(NAMESPACE, MessagingApp.class, artifactJar);
    final WorkerManager workerManager = appManager.getWorkerManager(MessagingApp.MessagingWorker.class.getSimpleName()).start();
    MessagingContext messagingContext = getMessagingContext();
    final MessagingAdmin messagingAdmin = getMessagingAdmin(NAMESPACE);
    // Wait for the worker to create the topic
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            try {
                messagingAdmin.getTopicProperties(MessagingApp.TOPIC);
                return true;
            } catch (TopicNotFoundException e) {
                return false;
            }
        }
    }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Publish a message
    String message = "message";
    MessagePublisher messagePublisher = messagingContext.getMessagePublisher();
    messagePublisher.publish(NAMESPACE.getNamespace(), MessagingApp.TOPIC, message);
    // The worker will publish back a message with payload as concat(message, message)
    final MessageFetcher messageFetcher = messagingContext.getMessageFetcher();
    Tasks.waitFor(message + message, new Callable<String>() {

        @Override
        public String call() throws Exception {
            try (CloseableIterator<Message> iterator = messageFetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, Integer.MAX_VALUE, 0L)) {
                Message message = Iterators.getLast(iterator, null);
                return message == null ? null : message.getPayloadAsString();
            }
        }
    }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Publish concat(message + message) to the app
    messagePublisher.publish(NAMESPACE.getNamespace(), MessagingApp.TOPIC, message + message);
    // timeout.
    try {
        Tasks.waitFor(message + message + message + message, new Callable<String>() {

            @Override
            public String call() throws Exception {
                try (CloseableIterator<Message> iterator = messageFetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, Integer.MAX_VALUE, 0L)) {
                    Message message = Iterators.getLast(iterator, null);
                    return message == null ? null : message.getPayloadAsString();
                }
            }
        }, 2, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        Assert.fail("Expected timeout exception");
    } catch (TimeoutException e) {
    // expected
    }
    // Now publish a message to the control topic, to unblock the transaction block.
    messagePublisher.publish(NAMESPACE.getNamespace(), MessagingApp.CONTROL_TOPIC, message);
    // Should expect a new message as concat(message, message, message, message)
    Tasks.waitFor(message + message + message + message, new Callable<String>() {

        @Override
        public String call() throws Exception {
            try (CloseableIterator<Message> iterator = messageFetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, Integer.MAX_VALUE, 0L)) {
                Message message = Iterators.getLast(iterator, null);
                return message == null ? null : message.getPayloadAsString();
            }
        }
    }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Wait for the worker to finish and verify that it completes successfully.
    workerManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.SECONDS);
}

Example 9 with CloseableIterator

use of co.cask.cdap.api.dataset.lib.CloseableIterator in project cdap by caskdata.

the class ObjectMappedTableDatasetTest method testScan.

@Test
public void testScan() throws Exception {
    dsFrameworkUtil.createInstance(ObjectMappedTable.class.getName(), RECORDS_ID, ObjectMappedTableProperties.builder().setType(Record.class).build());
    try {
        final ObjectMappedTableDataset<Record> records = dsFrameworkUtil.getInstance(RECORDS_ID);
        TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor((TransactionAware) records);
        Record record1 = new Record(Integer.MAX_VALUE, Long.MAX_VALUE, Float.MAX_VALUE, Double.MAX_VALUE, "foobar", Bytes.toBytes("foobar"), ByteBuffer.wrap(Bytes.toBytes("foobar")), UUID.randomUUID());
        Record record2 = new Record(Integer.MIN_VALUE, Long.MIN_VALUE, Float.MIN_VALUE, Double.MIN_VALUE, "baz", Bytes.toBytes("baz"), ByteBuffer.wrap(Bytes.toBytes("baz")), UUID.randomUUID());
        Record record3 = new Record(1, 0L, 3.14f, 3.14159265358979323846, "hello", Bytes.toBytes("world"), ByteBuffer.wrap(Bytes.toBytes("yo")), UUID.randomUUID());
        final List<KeyValue<byte[], Record>> recordList = Lists.newArrayList();
        recordList.add(new KeyValue<>(Bytes.toBytes("123"), record1));
        recordList.add(new KeyValue<>(Bytes.toBytes("456"), record2));
        recordList.add(new KeyValue<>(Bytes.toBytes("789"), record3));
        for (final KeyValue<byte[], Record> record : recordList) {
            txnl.execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    records.write(record.getKey(), record.getValue());
                }
            });
        }
        final List<KeyValue<byte[], Record>> actualList = Lists.newArrayList();
        txnl.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                CloseableIterator<KeyValue<byte[], Record>> results = records.scan((String) null, null);
                while (results.hasNext()) {
                    actualList.add(results.next());
                }
                results.close();
            }
        });
        Assert.assertEquals(recordList.size(), actualList.size());
        for (int i = 0; i < actualList.size(); i++) {
            KeyValue<byte[], Record> expected = recordList.get(i);
            KeyValue<byte[], Record> actual = actualList.get(i);
            Assert.assertArrayEquals(expected.getKey(), actual.getKey());
            Assert.assertEquals(expected.getValue(), actual.getValue());
        }
        txnl.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                CloseableIterator<KeyValue<byte[], Record>> results = records.scan("789", null);
                KeyValue<byte[], Record> actualRecord = results.next();
                Assert.assertFalse(results.hasNext());
                Assert.assertArrayEquals(actualRecord.getKey(), recordList.get(2).getKey());
                Assert.assertEquals(actualRecord.getValue(), recordList.get(2).getValue());
                results.close();
                results = records.scan(null, "124");
                actualRecord = results.next();
                Assert.assertFalse(results.hasNext());
                Assert.assertArrayEquals(actualRecord.getKey(), recordList.get(0).getKey());
                Assert.assertEquals(actualRecord.getValue(), recordList.get(0).getValue());
                results.close();
                results = records.scan(null, "123");
                Assert.assertFalse(results.hasNext());
                results.close();
            }
        });
        actualList.clear();
        txnl.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                Scan scan = new Scan(null, null);
                CloseableIterator<KeyValue<byte[], Record>> results = records.scan(scan);
                while (results.hasNext()) {
                    actualList.add(results.next());
                }
            }
        });
        Assert.assertEquals(recordList.size(), actualList.size());
    } finally {
        dsFrameworkUtil.deleteInstance(RECORDS_ID);
    }
}

Example 10 with CloseableIterator

use of co.cask.cdap.api.dataset.lib.CloseableIterator in project cdap by caskdata.

the class ObjectStoreDatasetTest method testScanObjectStore.

@Test
public void testScanObjectStore() throws Exception {
    DatasetId scan = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("scan");
    createObjectStoreInstance(scan, String.class);
    final ObjectStoreDataset<String> t = dsFrameworkUtil.getInstance(scan);
    TransactionExecutor txnl = dsFrameworkUtil.newTransactionExecutor(t);
    // write 10 values
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            for (int i = 0; i < 10; i++) {
                byte[] key = Bytes.toBytes(i);
                t.write(key, String.valueOf(i));
            }
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Iterator<KeyValue<byte[], String>> objectsIterator = t.scan(Bytes.toBytes(0), Bytes.toBytes(10));
            int sum = 0;
            while (objectsIterator.hasNext()) {
                sum += Integer.parseInt(objectsIterator.next().getValue());
            }
            //checking the sum equals sum of values from (0..9) which are the rows written and scanned for.
            Assert.assertEquals(45, sum);
        }
    });
    // start a transaction, scan part of them elements using scanner, close the scanner,
    // then call next() on scanner, it should fail
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            CloseableIterator<KeyValue<byte[], String>> objectsIterator = t.scan(Bytes.toBytes(0), Bytes.toBytes(10));
            int rowCount = 0;
            while (objectsIterator.hasNext() && (rowCount < 5)) {
                rowCount++;
            }
            objectsIterator.close();
            try {
                objectsIterator.next();
                Assert.fail("Reading after closing Scanner returned result.");
            } catch (NoSuchElementException e) {
            }
        }
    });
    dsFrameworkUtil.deleteInstance(scan);
}