Examples with StreamAdmin co.cask.cdap.data2.transaction.stream.StreamAdmin used on opensource projects

Example 21 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class FlowUtils method configureQueue.

/**
   * Configures all queues being used in a flow.
   *
   * @return A Multimap from flowletId to QueueName where the flowlet is a consumer of.
   */
public static Multimap<String, QueueName> configureQueue(Program program, FlowSpecification flowSpec, final StreamAdmin streamAdmin, QueueAdmin queueAdmin, TransactionExecutorFactory txExecutorFactory) {
    // Generate all queues specifications
    ApplicationId appId = new ApplicationId(program.getNamespaceId(), program.getApplicationId());
    Table<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> queueSpecs = new SimpleQueueSpecificationGenerator(appId).create(flowSpec);
    // For each queue in the flow, gather all consumer groups information
    Multimap<QueueName, ConsumerGroupConfig> queueConfigs = HashMultimap.create();
    // Loop through each flowlet and generate the map from consumer flowlet id to queue
    ImmutableSetMultimap.Builder<String, QueueName> resultBuilder = ImmutableSetMultimap.builder();
    for (Map.Entry<String, FlowletDefinition> entry : flowSpec.getFlowlets().entrySet()) {
        String flowletId = entry.getKey();
        for (QueueSpecification queueSpec : Iterables.concat(queueSpecs.column(flowletId).values())) {
            resultBuilder.put(flowletId, queueSpec.getQueueName());
        }
    }
    // For each queue, gather all consumer groups.
    for (QueueSpecification queueSpec : Iterables.concat(queueSpecs.values())) {
        QueueName queueName = queueSpec.getQueueName();
        queueConfigs.putAll(queueName, getAllConsumerGroups(program, flowSpec, queueName, queueSpecs));
    }
    try {
        // Configure each stream consumer in the Flow. Also collects all queue configurers.
        final List<ConsumerGroupConfigurer> groupConfigurers = Lists.newArrayList();
        final Map<StreamId, Map<Long, Integer>> streamConfigurers = Maps.newHashMap();
        for (Map.Entry<QueueName, Collection<ConsumerGroupConfig>> entry : queueConfigs.asMap().entrySet()) {
            LOG.info("Queue config for {} : {}", entry.getKey(), entry.getValue());
            if (entry.getKey().isStream()) {
                Map<Long, Integer> configs = Maps.newHashMap();
                for (ConsumerGroupConfig config : entry.getValue()) {
                    configs.put(config.getGroupId(), config.getGroupSize());
                }
                streamConfigurers.put(entry.getKey().toStreamId(), configs);
            } else {
                groupConfigurers.add(new ConsumerGroupConfigurer(queueAdmin.getQueueConfigurer(entry.getKey()), entry.getValue()));
            }
        }
        // Configure queue transactionally
        try {
            Transactions.createTransactionExecutor(txExecutorFactory, groupConfigurers).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    for (ConsumerGroupConfigurer configurer : groupConfigurers) {
                        configurer.configure();
                    }
                    for (Map.Entry<StreamId, Map<Long, Integer>> entry : streamConfigurers.entrySet()) {
                        streamAdmin.configureGroups(entry.getKey(), entry.getValue());
                    }
                }
            });
        } finally {
            for (ConsumerGroupConfigurer configurer : groupConfigurers) {
                Closeables.closeQuietly(configurer);
            }
        }
        return resultBuilder.build();
    } catch (Exception e) {
        LOG.error("Failed to configure queues", e);
        throw Throwables.propagate(e);
    }
}

Example 22 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class EntityExistenceTest method setup.

@BeforeClass
public static void setup() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.INSTANCE_NAME, EXISTS);
    Injector injector = AppFabricTestHelper.getInjector(cConf);
    NamespaceStore nsStore = injector.getInstance(NamespaceStore.class);
    ArtifactRepository artifactRepository = injector.getInstance(ArtifactRepository.class);
    cConf = injector.getInstance(CConfiguration.class);
    nsStore.create(new NamespaceMeta.Builder().setName(EXISTS).build());
    existenceVerifier = injector.getInstance(EntityExistenceVerifier.class);
    LocalLocationFactory lf = new LocalLocationFactory(TEMPORARY_FOLDER.newFolder());
    File artifactFile = new File(AppJarHelper.createDeploymentJar(lf, AllProgramsApp.class).toURI());
    artifactRepository.addArtifact(ARTIFACT.toId(), artifactFile);
    AppFabricTestHelper.deployApplication(NAMESPACE.toId(), AllProgramsApp.class, null, cConf);
    StreamAdmin streamAdmin = injector.getInstance(StreamAdmin.class);
    streamAdmin.createOrUpdateView(VIEW, new ViewSpecification(new FormatSpecification("csv", null)));
}

Example 23 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class StreamConsumerTestBase method testTTL.

@Test
public void testTTL() throws Exception {
    String stream = "testTTL";
    StreamId streamId = TEST_NAMESPACE.stream(stream);
    StreamAdmin streamAdmin = getStreamAdmin();
    // Create stream with ttl of 1 day
    final long ttl = TimeUnit.DAYS.toMillis(1);
    final long currentTime = System.currentTimeMillis();
    final long increment = TimeUnit.SECONDS.toMillis(1);
    final long approxEarliestNonExpiredTime = currentTime - TimeUnit.HOURS.toMillis(1);
    Properties streamProperties = new Properties();
    streamProperties.setProperty(Constants.Stream.TTL, Long.toString(ttl));
    streamProperties.setProperty(Constants.Stream.PARTITION_DURATION, Long.toString(ttl));
    streamAdmin.create(streamId, streamProperties);
    StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    streamAdmin.configureInstances(streamId, 0L, 1);
    StreamConsumerFactory consumerFactory = getConsumerFactory();
    Assert.assertEquals(ttl, streamConfig.getTTL());
    Assert.assertEquals(ttl, streamConfig.getPartitionDuration());
    Set<StreamEvent> expectedEvents = Sets.newTreeSet(STREAM_EVENT_COMPARATOR);
    FileWriter<StreamEvent> writer = getFileWriterFactory().create(streamConfig, 0);
    try {
        // Write 10 expired messages
        writeEvents(streamConfig, "Old event ", 20, new IncrementingClock(0, 1));
        // Write 5 non-expired messages
        expectedEvents.addAll(writeEvents(streamConfig, "New event ", 12, new IncrementingClock(approxEarliestNonExpiredTime, increment)));
    } finally {
        writer.close();
    }
    // Dequeue from stream. Should only get the 5 unexpired events.
    StreamConsumer consumer = consumerFactory.create(streamId, stream, new ConsumerConfig(0L, 0, 1, DequeueStrategy.FIFO, null));
    try {
        verifyEvents(consumer, expectedEvents);
        TransactionContext txContext = createTxContext(consumer);
        txContext.start();
        try {
            // Should be no more pending events
            DequeueResult<StreamEvent> result = consumer.poll(1, 2, TimeUnit.SECONDS);
            Assert.assertTrue(result.isEmpty());
        } finally {
            txContext.finish();
        }
    } finally {
        consumer.close();
    }
}

Example 24 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class HBaseConsumerStateTest method init.

@BeforeClass
public static void init() throws Exception {
    zkServer = InMemoryZKServer.builder().setDataDir(TMP_FOLDER.newFolder()).build();
    zkServer.startAndWait();
    Configuration hConf = TEST_HBASE.getConfiguration();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TMP_FOLDER.newFolder().getAbsolutePath());
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new ZKClientModule(), new NonCustomLocationUnitTestModule().getModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new TransactionMetricsModule(), new DataFabricModules().getDistributedModules(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
        }
    }, new DataSetsModules().getInMemoryModules(), new SystemDatasetRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new StreamAdminModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
            bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }));
    zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    streamAdmin = injector.getInstance(StreamAdmin.class);
    stateStoreFactory = injector.getInstance(StreamConsumerStateStoreFactory.class);
    tableUtil = injector.getInstance(HBaseTableUtil.class);
    ddlExecutor = new HBaseDDLExecutorFactory(cConf, TEST_HBASE.getHBaseAdmin().getConfiguration()).get();
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(TEST_NAMESPACE));
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(OTHER_NAMESPACE));
    setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
}

Example 25 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class HBaseStreamConsumerTest method init.

@BeforeClass
public static void init() throws Exception {
    zkServer = InMemoryZKServer.builder().setDataDir(TMP_FOLDER.newFolder()).build();
    zkServer.startAndWait();
    Configuration hConf = TEST_HBASE.getConfiguration();
    cConf.setInt(Constants.Stream.CONTAINER_INSTANCES, 1);
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TMP_FOLDER.newFolder().getAbsolutePath());
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new ZKClientModule(), new NonCustomLocationUnitTestModule().getModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new TransactionMetricsModule(), new DataSetsModules().getInMemoryModules(), new SystemDatasetRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new DataFabricModules().getDistributedModules(), new StreamAdminModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(TransactionStateStorage.class).to(NoOpTransactionStateStorage.class);
            bind(TransactionSystemClient.class).to(InMemoryTxSystemClient.class).in(Singleton.class);
            bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
            bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }));
    zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    streamAdmin = injector.getInstance(StreamAdmin.class);
    consumerFactory = injector.getInstance(StreamConsumerFactory.class);
    txClient = injector.getInstance(TransactionSystemClient.class);
    txManager = TxInMemory.getTransactionManager(txClient);
    queueClientFactory = injector.getInstance(QueueClientFactory.class);
    fileWriterFactory = injector.getInstance(StreamFileWriterFactory.class);
    txManager.startAndWait();
    tableUtil = injector.getInstance(HBaseTableUtil.class);
    ddlExecutor = new HBaseDDLExecutorFactory(cConf, TEST_HBASE.getHBaseAdmin().getConfiguration()).get();
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(NamespaceId.SYSTEM));
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(TEST_NAMESPACE));
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(OTHER_NAMESPACE));
    setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
}