Examples with ConsumerConfig co.cask.cdap.data2.queue.ConsumerConfig used on opensource projects

Example 16 with ConsumerConfig

use of co.cask.cdap.data2.queue.ConsumerConfig in project cdap by caskdata.

the class StreamConsumerTestBase method testFIFORollback.

@Test
public void testFIFORollback() throws Exception {
    String stream = "testFIFORollback";
    StreamId streamId = TEST_NAMESPACE.stream(stream);
    StreamAdmin streamAdmin = getStreamAdmin();
    streamAdmin.create(streamId);
    StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    // Writes 5 events
    writeEvents(streamConfig, "Testing ", 5);
    streamAdmin.configureInstances(streamId, 0L, 2);
    StreamConsumerFactory consumerFactory = getConsumerFactory();
    StreamConsumer consumer0 = consumerFactory.create(streamId, "fifo.rollback", new ConsumerConfig(0L, 0, 2, DequeueStrategy.FIFO, null));
    StreamConsumer consumer1 = consumerFactory.create(streamId, "fifo.rollback", new ConsumerConfig(0L, 1, 2, DequeueStrategy.FIFO, null));
    // Try to dequeue using both consumers
    TransactionContext context0 = createTxContext(consumer0);
    TransactionContext context1 = createTxContext(consumer1);
    context0.start();
    context1.start();
    DequeueResult<StreamEvent> result0 = consumer0.poll(1, 1, TimeUnit.SECONDS);
    DequeueResult<StreamEvent> result1 = consumer1.poll(1, 1, TimeUnit.SECONDS);
    Assert.assertEquals("Testing 0", Charsets.UTF_8.decode(result0.iterator().next().getBody()).toString());
    Assert.assertEquals("Testing 1", Charsets.UTF_8.decode(result1.iterator().next().getBody()).toString());
    // Commit the first one, rollback the second one.
    context0.finish();
    context1.abort();
    // Dequeue again with the consuemrs
    context0.start();
    context1.start();
    result0 = consumer0.poll(1, 1, TimeUnit.SECONDS);
    result1 = consumer1.poll(1, 1, TimeUnit.SECONDS);
    // Expect consumer 0 keep proceeding while consumer 1 will retry with what it claimed in previous transaction.
    // This is the optimization in FIFO mode to avoid going back and rescanning.
    Assert.assertEquals("Testing 2", Charsets.UTF_8.decode(result0.iterator().next().getBody()).toString());
    Assert.assertEquals("Testing 1", Charsets.UTF_8.decode(result1.iterator().next().getBody()).toString());
    // Commit both
    context0.finish();
    context1.finish();
    consumer0.close();
    consumer1.close();
}

Example 17 with ConsumerConfig

use of co.cask.cdap.data2.queue.ConsumerConfig in project cdap by caskdata.

the class StreamConsumerTestBase method testTTLStartingFile.

@Category(SlowTests.class)
@Test
public void testTTLStartingFile() throws Exception {
    String stream = "testTTLStartingFile";
    StreamId streamId = TEST_NAMESPACE.stream(stream);
    StreamAdmin streamAdmin = getStreamAdmin();
    // Create stream with ttl of 3 seconds and partition duration of 3 seconds
    final long ttl = TimeUnit.SECONDS.toMillis(3);
    Properties streamProperties = new Properties();
    streamProperties.setProperty(Constants.Stream.TTL, Long.toString(ttl));
    streamProperties.setProperty(Constants.Stream.PARTITION_DURATION, Long.toString(ttl));
    streamAdmin.create(streamId, streamProperties);
    StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    streamAdmin.configureGroups(streamId, ImmutableMap.of(0L, 1, 1L, 1));
    StreamConsumerFactory consumerFactory = getConsumerFactory();
    StreamConsumer consumer = consumerFactory.create(streamId, stream, new ConsumerConfig(0L, 0, 1, DequeueStrategy.FIFO, null));
    StreamConsumer newConsumer;
    Set<StreamEvent> expectedEvents = Sets.newTreeSet(STREAM_EVENT_COMPARATOR);
    try {
        // Create a new consumer for second consumer verification.
        // Need to create consumer before write event because in HBase, creation of consumer took couple seconds.
        newConsumer = consumerFactory.create(streamId, stream, new ConsumerConfig(1L, 0, 1, DequeueStrategy.FIFO, null));
        // write 20 events in a partition that will be expired due to sleeping the TTL
        writeEvents(streamConfig, "Phase 0 expired event ", 20);
        Thread.sleep(ttl);
        verifyEvents(consumer, expectedEvents);
        // also verify for a new consumer
        try {
            verifyEvents(newConsumer, expectedEvents);
        } finally {
            newConsumer.close();
        }
        // Create a new consumer for second consumer verification (with clean state)
        // Need to create consumer before write event because in HBase, creation of consumer took couple seconds.
        streamAdmin.configureGroups(streamId, ImmutableMap.of(0L, 1));
        streamAdmin.configureGroups(streamId, ImmutableMap.of(0L, 1, 1L, 1));
        newConsumer = consumerFactory.create(streamId, stream, new ConsumerConfig(1L, 0, 1, DequeueStrategy.FIFO, null));
        // write 20 events in a partition and read it back immediately. They shouldn't expired.
        expectedEvents.addAll(writeEvents(streamConfig, "Phase 1 non-expired event ", 20));
        verifyEvents(consumer, expectedEvents);
        // also verify for a new consumer
        try {
            verifyEvents(newConsumer, expectedEvents);
        } finally {
            newConsumer.close();
        }
        // Create a new consumer for second consumer verification (with clean state)
        // Need to create consumer before write event because in HBase, creation of consumer took couple seconds.
        streamAdmin.configureGroups(streamId, ImmutableMap.of(0L, 1));
        streamAdmin.configureGroups(streamId, ImmutableMap.of(0L, 1, 1L, 1));
        newConsumer = consumerFactory.create(streamId, stream, new ConsumerConfig(1L, 0, 1, DequeueStrategy.FIFO, null));
        // write 20 events in a partition that will be expired due to sleeping the TTL
        // This will write to a new partition different then the first batch write.
        // Also, because it sleep TTL time, the previous batch write would also get expired.
        expectedEvents.clear();
        writeEvents(streamConfig, "Phase 2 expired event ", 20);
        Thread.sleep(ttl);
        verifyEvents(consumer, expectedEvents);
        // also verify for a new consumer
        try {
            verifyEvents(newConsumer, expectedEvents);
        } finally {
            newConsumer.close();
        }
        // Create a new consumer for second consumer verification (with clean state)
        // Need to create consumer before write event because in HBase, creation of consumer took couple seconds.
        streamAdmin.configureGroups(streamId, ImmutableMap.of(0L, 1));
        streamAdmin.configureGroups(streamId, ImmutableMap.of(0L, 1, 1L, 1));
        newConsumer = consumerFactory.create(streamId, stream, new ConsumerConfig(1L, 0, 1, DequeueStrategy.FIFO, null));
        // write 20 events in a partition and read it back immediately. They shouldn't expire.
        expectedEvents.addAll(writeEvents(streamConfig, "Phase 3 non-expired event ", 20));
        verifyEvents(consumer, expectedEvents);
        // also verify for a new consumer
        try {
            verifyEvents(newConsumer, expectedEvents);
        } finally {
            newConsumer.close();
        }
        // Should be no more pending events
        expectedEvents.clear();
        verifyEvents(consumer, expectedEvents);
    } finally {
        consumer.close();
    }
}

Example 18 with ConsumerConfig

use of co.cask.cdap.data2.queue.ConsumerConfig in project cdap by caskdata.

the class HBaseQueueDebugger method scanQueue.

private void scanQueue(TransactionExecutor txExecutor, HBaseConsumerStateStore stateStore, QueueName queueName, QueueBarrier start, @Nullable QueueBarrier end, final QueueStatistics outStats) throws Exception {
    final byte[] queueRowPrefix = QueueEntryRow.getQueueRowPrefix(queueName);
    ConsumerGroupConfig groupConfig = start.getGroupConfig();
    printProgress("Got consumer group config: %s\n", groupConfig);
    HBaseQueueAdmin admin = queueClientFactory.getQueueAdmin();
    TableId tableId = admin.getDataTableId(queueName, QueueConstants.QueueType.SHARDED_QUEUE);
    HTable hTable = queueClientFactory.createHTable(tableId);
    printProgress("Looking at HBase table: %s\n", Bytes.toString(hTable.getTableName()));
    final byte[] stateColumnName = Bytes.add(QueueEntryRow.STATE_COLUMN_PREFIX, Bytes.toBytes(groupConfig.getGroupId()));
    int distributorBuckets = queueClientFactory.getDistributorBuckets(hTable.getTableDescriptor());
    ShardedHBaseQueueStrategy queueStrategy = new ShardedHBaseQueueStrategy(tableUtil, distributorBuckets);
    ScanBuilder scan = tableUtil.buildScan();
    scan.setStartRow(start.getStartRow());
    if (end != null) {
        scan.setStopRow(end.getStartRow());
    } else {
        scan.setStopRow(QueueEntryRow.getQueueEntryRowKey(queueName, Long.MAX_VALUE, Integer.MAX_VALUE));
    }
    // Needs to include meta column for row that doesn't have state yet.
    scan.addColumn(QueueEntryRow.COLUMN_FAMILY, QueueEntryRow.META_COLUMN);
    scan.addColumn(QueueEntryRow.COLUMN_FAMILY, stateColumnName);
    // Don't do block cache for debug tool. We don't want old blocks get cached
    scan.setCacheBlocks(false);
    scan.setMaxVersions(1);
    printProgress("Scanning section with scan: %s\n", scan.toString());
    List<Integer> instanceIds = Lists.newArrayList();
    if (groupConfig.getDequeueStrategy() == DequeueStrategy.FIFO) {
        instanceIds.add(0);
    } else {
        for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
            instanceIds.add(instanceId);
        }
    }
    final int rowsCache = Integer.parseInt(System.getProperty(PROP_ROWS_CACHE, "100000"));
    for (final int instanceId : instanceIds) {
        printProgress("Processing instance %d", instanceId);
        ConsumerConfig consConfig = new ConsumerConfig(groupConfig, instanceId);
        final QueueScanner scanner = queueStrategy.createScanner(consConfig, hTable, scan.build(), rowsCache);
        try {
            txExecutor.execute(new TransactionExecutor.Procedure<HBaseConsumerStateStore>() {

                @Override
                public void apply(HBaseConsumerStateStore input) throws Exception {
                    ImmutablePair<byte[], Map<byte[], byte[]>> result;
                    while ((result = scanner.next()) != null) {
                        byte[] rowKey = result.getFirst();
                        Map<byte[], byte[]> columns = result.getSecond();
                        visitRow(outStats, input.getTransaction(), rowKey, columns.get(stateColumnName), queueRowPrefix.length);
                        if (showProgress() && outStats.getTotal() % rowsCache == 0) {
                            System.out.printf("\rProcessing instance %d: %s", instanceId, outStats.getReport(showTxTimestampOnly()));
                        }
                    }
                }
            }, stateStore);
        } catch (TransactionFailureException e) {
            // Ignore transaction not in progress exception as it's caused by short TX timeout on commit
            if (!(Throwables.getRootCause(e) instanceof TransactionNotInProgressException)) {
                throw Throwables.propagate(e);
            }
        }
        printProgress("\rProcessing instance %d: %s\n", instanceId, outStats.getReport(showTxTimestampOnly()));
    }
}

Example 19 with ConsumerConfig

use of co.cask.cdap.data2.queue.ConsumerConfig in project cdap by caskdata.

the class DequeueScanObserver method preScannerOpen.

@Override
public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s) throws IOException {
    ConsumerConfig consumerConfig = DequeueScanAttributes.getConsumerConfig(scan);
    Transaction tx = DequeueScanAttributes.getTx(scan);
    if (consumerConfig == null || tx == null) {
        return super.preScannerOpen(e, scan, s);
    }
    Filter dequeueFilter = new DequeueFilter(consumerConfig, tx);
    Filter existing = scan.getFilter();
    if (existing != null) {
        Filter combined = new FilterList(FilterList.Operator.MUST_PASS_ALL, existing, dequeueFilter);
        scan.setFilter(combined);
    } else {
        scan.setFilter(dequeueFilter);
    }
    return super.preScannerOpen(e, scan, s);
}

Example 20 with ConsumerConfig

use of co.cask.cdap.data2.queue.ConsumerConfig in project cdap by caskdata.

the class DequeueFilter method filterRowCells.

@Override
public void filterRowCells(List<Cell> cells) {
    byte[] dataBytes = null;
    byte[] metaBytes = null;
    byte[] stateBytes = null;
    // list is very short so it is ok to loop thru to find columns
    for (Cell cell : cells) {
        if (CellUtil.matchingQualifier(cell, QueueEntryRow.DATA_COLUMN)) {
            dataBytes = CellUtil.cloneValue(cell);
        } else if (CellUtil.matchingQualifier(cell, QueueEntryRow.META_COLUMN)) {
            metaBytes = CellUtil.cloneValue(cell);
        } else if (CellUtil.matchingQualifier(cell, stateColumnName)) {
            stateBytes = CellUtil.cloneValue(cell);
        }
    }
    if (dataBytes == null || metaBytes == null) {
        skipRow = true;
        return;
    }
    QueueEntryRow.CanConsume canConsume = QueueEntryRow.canConsume(consumerConfig, transaction, writePointer, counter, metaBytes, stateBytes);
    // Only skip the row when canConsumer == NO, so that in case of NO_INCLUDING_ALL_OLDER, the client
    // can still see the row and move the scan start row.
    skipRow = canConsume == QueueEntryRow.CanConsume.NO;
}