Examples with StreamAdmin co.cask.cdap.data2.transaction.stream.StreamAdmin used on opensource projects

Example 16 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class DFSStreamHeartbeatsTest method beforeClass.

@BeforeClass
public static void beforeClass() throws IOException {
    zkServer = InMemoryZKServer.builder().setDataDir(TEMP_FOLDER.newFolder()).build();
    zkServer.startAndWait();
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    cConf.setInt(Constants.Stream.CONTAINER_INSTANCE_ID, 0);
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    Injector injector = Guice.createInjector(Modules.override(new ZKClientModule(), new DataFabricModules().getInMemoryModules(), new ConfigModule(cConf, new Configuration()), new DiscoveryRuntimeModule().getInMemoryModules(), new NonCustomLocationUnitTestModule().getModule(), new ExploreClientModule(), new DataSetServiceModules().getInMemoryModules(), new DataSetsModules().getStandaloneModules(), new NotificationFeedServiceRuntimeModule().getInMemoryModules(), new NotificationServiceRuntimeModule().getInMemoryModules(), new MetricsClientRuntimeModule().getInMemoryModules(), new ViewAdminModules().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule(), // that performs heartbeats aggregation
    new StreamServiceRuntimeModule().getDistributedModules(), new StreamAdminModules().getInMemoryModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class);
            install(new NamespaceClientRuntimeModule().getInMemoryModules());
            bind(StreamConsumerStateStoreFactory.class).to(LevelDBStreamConsumerStateStoreFactory.class).in(Singleton.class);
            bind(StreamAdmin.class).to(FileStreamAdmin.class).in(Singleton.class);
            bind(StreamConsumerFactory.class).to(LevelDBStreamFileConsumerFactory.class).in(Singleton.class);
            bind(StreamFileWriterFactory.class).to(LocationStreamFileWriterFactory.class).in(Singleton.class);
            bind(StreamFileJanitorService.class).to(LocalStreamFileJanitorService.class).in(Scopes.SINGLETON);
            bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class).in(Scopes.SINGLETON);
            bind(HeartbeatPublisher.class).to(MockHeartbeatPublisher.class).in(Scopes.SINGLETON);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }));
    zkClient = injector.getInstance(ZKClientService.class);
    txManager = injector.getInstance(TransactionManager.class);
    datasetService = injector.getInstance(DatasetService.class);
    notificationService = injector.getInstance(NotificationService.class);
    streamHttpService = injector.getInstance(StreamHttpService.class);
    streamService = injector.getInstance(StreamService.class);
    heartbeatPublisher = (MockHeartbeatPublisher) injector.getInstance(HeartbeatPublisher.class);
    namespacedLocationFactory = injector.getInstance(NamespacedLocationFactory.class);
    streamAdmin = injector.getInstance(StreamAdmin.class);
    zkClient.startAndWait();
    txManager.startAndWait();
    datasetService.startAndWait();
    notificationService.startAndWait();
    streamHttpService.startAndWait();
    streamService.startAndWait();
    hostname = streamHttpService.getBindAddress().getHostName();
    port = streamHttpService.getBindAddress().getPort();
    Locations.mkdirsIfNotExists(namespacedLocationFactory.get(NamespaceId.DEFAULT));
}

Example 17 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class StreamFileSizeFetcherTest method testFetchSize.

@Test
public void testFetchSize() throws Exception {
    final String streamName = "testFetchSize";
    StreamId streamId = NamespaceId.DEFAULT.stream(streamName);
    final int nbEvents = 100;
    StreamAdmin streamAdmin = new TestStreamAdmin(namespacedLocationFactory, Long.MAX_VALUE, 1000);
    streamAdmin.create(streamId);
    StreamConfig config = streamAdmin.getConfig(streamId);
    try {
        StreamUtils.fetchStreamFilesSize(StreamUtils.createGenerationLocation(config.getLocation(), StreamUtils.getGeneration(config)));
        Assert.fail("No stream file created yet");
    } catch (IOException e) {
    // Expected
    }
    // Creates a stream file that has no event inside
    Location partitionLocation = StreamUtils.createPartitionLocation(config.getLocation(), 0, Long.MAX_VALUE);
    Location dataLocation = StreamUtils.createStreamLocation(partitionLocation, "writer", 0, StreamFileType.EVENT);
    Location idxLocation = StreamUtils.createStreamLocation(partitionLocation, "writer", 0, StreamFileType.INDEX);
    StreamDataFileWriter writer = new StreamDataFileWriter(Locations.newOutputSupplier(dataLocation), Locations.newOutputSupplier(idxLocation), 10000L);
    // Write 100 events to the stream
    for (int i = 0; i < nbEvents; i++) {
        writer.append(StreamFileTestUtils.createEvent(i, "foo"));
    }
    writer.close();
    long size = StreamUtils.fetchStreamFilesSize(StreamUtils.createGenerationLocation(config.getLocation(), StreamUtils.getGeneration(config)));
    Assert.assertTrue(size > 0);
    Assert.assertEquals(dataLocation.length(), size);
}

Example 18 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class StreamFileJanitorTestBase method testCleanupGeneration.

@Test
public void testCleanupGeneration() throws Exception {
    // Create a stream and performs couple truncate
    String streamName = "testCleanupGeneration";
    StreamId streamId = NamespaceId.DEFAULT.stream(streamName);
    StreamAdmin streamAdmin = getStreamAdmin();
    streamAdmin.create(streamId);
    StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    StreamFileJanitor janitor = new StreamFileJanitor(getCConfiguration(), getStreamAdmin(), getNamespacedLocationFactory(), getNamespaceAdmin(), impersonator);
    for (int i = 0; i < 5; i++) {
        FileWriter<StreamEvent> writer = createWriter(streamId);
        writer.append(StreamFileTestUtils.createEvent(System.currentTimeMillis(), "Testing"));
        writer.close();
        // Call cleanup before truncate. The current generation should stand.
        janitor.clean(streamConfig.getLocation(), streamConfig.getTTL(), System.currentTimeMillis());
        verifyGeneration(streamConfig, i);
        streamAdmin.truncate(streamId);
    }
    int generation = StreamUtils.getGeneration(streamConfig);
    Assert.assertEquals(5, generation);
    janitor.clean(streamConfig.getLocation(), streamConfig.getTTL(), System.currentTimeMillis());
    // Verify the stream directory should only contains the generation directory
    for (Location location : streamConfig.getLocation().list()) {
        if (location.isDirectory()) {
            Assert.assertEquals(generation, Integer.parseInt(location.getName()));
        }
    }
}

Example 19 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class AbstractStreamFileConsumerFactory method create.

@Override
public final StreamConsumer create(StreamId streamId, String namespace, ConsumerConfig consumerConfig) throws IOException {
    StreamConfig streamConfig = StreamUtils.ensureExists(streamAdmin, streamId);
    TableId tableId = getTableId(streamId, namespace);
    StreamConsumerStateStore stateStore = stateStoreFactory.create(streamConfig);
    StreamConsumerState consumerState = stateStore.get(consumerConfig.getGroupId(), consumerConfig.getInstanceId());
    return create(tableId, streamConfig, consumerConfig, stateStore, consumerState, createReader(streamConfig, consumerState), new TTLReadFilter(streamConfig.getTTL()));
}

Example 20 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class ContextManager method createContext.

// this method is called by the mappers/reducers of jobs launched by Hive.
private static Context createContext(Configuration conf) throws IOException {
    // Create context needs to happen only when running in as a MapReduce job.
    // In other cases, ContextManager will be initialized using saveContext method.
    CConfiguration cConf = ConfigurationUtil.get(conf, Constants.Explore.CCONF_KEY, CConfCodec.INSTANCE);
    Configuration hConf = ConfigurationUtil.get(conf, Constants.Explore.HCONF_KEY, HConfCodec.INSTANCE);
    Injector injector = createInjector(cConf, hConf);
    ZKClientService zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
    StreamAdmin streamAdmin = injector.getInstance(StreamAdmin.class);
    SystemDatasetInstantiatorFactory datasetInstantiatorFactory = injector.getInstance(SystemDatasetInstantiatorFactory.class);
    AuthenticationContext authenticationContext = injector.getInstance(AuthenticationContext.class);
    AuthorizationEnforcer authorizationEnforcer = injector.getInstance(AuthorizationEnforcer.class);
    return new Context(datasetFramework, streamAdmin, zkClientService, datasetInstantiatorFactory, authenticationContext, authorizationEnforcer);
}