Examples with HBaseQueueAdmin co.cask.cdap.data2.transaction.queue.hbase.HBaseQueueAdmin used on opensource projects

Example 6 with HBaseQueueAdmin

use of co.cask.cdap.data2.transaction.queue.hbase.HBaseQueueAdmin in project cdap by caskdata.

the class HBaseQueueTest method forceEviction.

@Override
protected void forceEviction(QueueName queueName, int numGroups) throws Exception {
    TableId tableId = ((HBaseQueueAdmin) queueAdmin).getDataTableId(queueName);
    byte[] tableName = tableUtil.getHTableDescriptor(hbaseAdmin, tableId).getName();
    // make sure consumer config cache is updated with the latest tx snapshot
    takeTxSnapshot();
    final Class coprocessorClass = tableUtil.getQueueRegionObserverClassForVersion();
    TEST_HBASE.forEachRegion(tableName, new Function<HRegion, Object>() {

        public Object apply(HRegion region) {
            try {
                Coprocessor cp = region.getCoprocessorHost().findCoprocessor(coprocessorClass.getName());
                // calling cp.updateCache(), NOTE: cannot do normal cast and stuff because cp is loaded
                // by different classloader (corresponds to a cp's jar)
                LOG.info("forcing update of transaction state cache for HBaseQueueRegionObserver of region: {}", region);
                Method getTxStateCache = cp.getClass().getDeclaredMethod("getTxStateCache");
                getTxStateCache.setAccessible(true);
                Object txStateCache = getTxStateCache.invoke(cp);
                // the one returned is of type DefaultTransactionStateCache.
                // The refreshState method is a private method of its parent, TransactionStateCache
                Method refreshState = txStateCache.getClass().getSuperclass().getDeclaredMethod("refreshState");
                refreshState.setAccessible(true);
                refreshState.invoke(txStateCache);
                LOG.info("forcing update cache for HBaseQueueRegionObserver of region: {}", region);
                Method updateCache = cp.getClass().getDeclaredMethod("updateCache");
                updateCache.setAccessible(true);
                updateCache.invoke(cp);
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
            return null;
        }
    });
    // Force a table flush to trigger eviction
    TEST_HBASE.forceRegionFlush(tableName);
    TEST_HBASE.forceRegionCompact(tableName, true);
}

Example 7 with HBaseQueueAdmin

use of co.cask.cdap.data2.transaction.queue.hbase.HBaseQueueAdmin in project cdap by caskdata.

the class HBaseQueueTest method testQueueUpgrade.

// This test upgrade from old queue (salted base) to new queue (sharded base)
@Test(timeout = 30000L)
public void testQueueUpgrade() throws Exception {
    final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "upgrade");
    HBaseQueueAdmin hbaseQueueAdmin = (HBaseQueueAdmin) queueAdmin;
    HBaseQueueClientFactory hBaseQueueClientFactory = (HBaseQueueClientFactory) queueClientFactory;
    // Create the old queue table explicitly
    HBaseQueueAdmin oldQueueAdmin = new HBaseQueueAdmin(hConf, cConf, injector.getInstance(LocationFactory.class), injector.getInstance(HBaseTableUtil.class), injector.getInstance(DatasetFramework.class), injector.getInstance(TransactionExecutorFactory.class), QueueConstants.QueueType.QUEUE, injector.getInstance(NamespaceQueryAdmin.class), injector.getInstance(Impersonator.class));
    oldQueueAdmin.create(queueName);
    int buckets = cConf.getInt(QueueConstants.ConfigKeys.QUEUE_TABLE_PRESPLITS);
    try (final HBaseQueueProducer oldProducer = hBaseQueueClientFactory.createProducer(oldQueueAdmin, queueName, QueueConstants.QueueType.QUEUE, QueueMetrics.NOOP_QUEUE_METRICS, new SaltedHBaseQueueStrategy(tableUtil, buckets), new ArrayList<ConsumerGroupConfig>())) {
        // Enqueue 10 items to old queue table
        Transactions.createTransactionExecutor(executorFactory, oldProducer).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                for (int i = 0; i < 10; i++) {
                    oldProducer.enqueue(new QueueEntry("key", i, Bytes.toBytes("Message " + i)));
                }
            }
        });
    }
    // Configure the consumer
    final ConsumerConfig consumerConfig = new ConsumerConfig(0L, 0, 1, DequeueStrategy.HASH, "key");
    try (QueueConfigurer configurer = queueAdmin.getQueueConfigurer(queueName)) {
        Transactions.createTransactionExecutor(executorFactory, configurer).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                configurer.configureGroups(ImmutableList.of(consumerConfig));
            }
        });
    }
    // explicit set the consumer state to be the lowest start row
    try (HBaseConsumerStateStore stateStore = hbaseQueueAdmin.getConsumerStateStore(queueName)) {
        Transactions.createTransactionExecutor(executorFactory, stateStore).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                stateStore.updateState(consumerConfig.getGroupId(), consumerConfig.getInstanceId(), QueueEntryRow.getQueueEntryRowKey(queueName, 0L, 0));
            }
        });
    }
    // Enqueue 10 more items to new queue table
    createEnqueueRunnable(queueName, 10, 1, null).run();
    // Verify both old and new table have 10 rows each
    Assert.assertEquals(10, countRows(hbaseQueueAdmin.getDataTableId(queueName, QueueConstants.QueueType.QUEUE)));
    Assert.assertEquals(10, countRows(hbaseQueueAdmin.getDataTableId(queueName, QueueConstants.QueueType.SHARDED_QUEUE)));
    // Create a consumer. It should see all 20 items
    final List<String> messages = Lists.newArrayList();
    try (final QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
        while (messages.size() != 20) {
            Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    DequeueResult<byte[]> result = consumer.dequeue(20);
                    for (byte[] data : result) {
                        messages.add(Bytes.toString(data));
                    }
                }
            });
        }
    }
    verifyQueueIsEmpty(queueName, ImmutableList.of(consumerConfig));
}