Examples with LocationFactory org.apache.twill.filesystem.LocationFactory used on opensource projects

Example 6 with LocationFactory

use of org.apache.twill.filesystem.LocationFactory in project cdap by caskdata.

the class LocalConcurrentStreamWriterTest method init.

@BeforeClass
public static void init() throws IOException {
    LocationFactory locationFactory = new LocalLocationFactory(TMP_FOLDER.newFolder());
    namespacedLocationFactory = new NamespacedLocationFactoryTestClient(CConfiguration.create(), locationFactory);
}

Example 7 with LocationFactory

use of org.apache.twill.filesystem.LocationFactory in project cdap by caskdata.

the class DistributedStreamCoordinatorClientTest method init.

@BeforeClass
public static void init() throws Exception {
    zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
    zkServer.startAndWait();
    Configuration hConf = new Configuration();
    hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tmpFolder.newFolder().getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
    dfsCluster.waitClusterUp();
    final LocationFactory lf = new FileContextLocationFactory(dfsCluster.getFileSystem().getConf());
    final NamespacedLocationFactory nlf = new NamespacedLocationFactoryTestClient(cConf, lf);
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new ZKClientModule(), new DiscoveryRuntimeModule().getDistributedModules(), new DataFabricModules().getDistributedModules(), Modules.override(new DataSetsModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetadataStore.class).to(NoOpMetadataStore.class);
            // bind to an in mem implementation for this test since the DefaultOwnerStore uses transaction and in this
            // test we are not starting a transaction service
            bind(OwnerStore.class).to(InMemoryOwnerStore.class).in(Scopes.SINGLETON);
        }
    }), new TransactionMetricsModule(), new NotificationFeedServiceRuntimeModule().getInMemoryModules(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(LocationFactory.class).toInstance(lf);
            bind(NamespacedLocationFactory.class).toInstance(nlf);
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }, new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), Modules.override(new StreamAdminModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
        }
    }), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule());
    zkClient = injector.getInstance(ZKClientService.class);
    zkClient.startAndWait();
    setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
    streamAdmin = injector.getInstance(StreamAdmin.class);
    coordinatorClient = injector.getInstance(StreamCoordinatorClient.class);
    coordinatorClient.startAndWait();
}

Example 8 with LocationFactory

use of org.apache.twill.filesystem.LocationFactory in project cdap by caskdata.

the class StreamAdminTest method addCConfProperties.

protected static void addCConfProperties(CConfiguration cConf) throws IOException {
    File rootLocationFactoryPath = TEMPORARY_FOLDER.newFolder();
    cConf.setBoolean(Constants.Security.ENABLED, true);
    cConf.setBoolean(Constants.Security.Authorization.ENABLED, true);
    cConf.setBoolean(Constants.Security.KERBEROS_ENABLED, false);
    cConf.setInt(Constants.Security.Authorization.CACHE_MAX_ENTRIES, 0);
    LocationFactory locationFactory = new LocalLocationFactory(rootLocationFactoryPath);
    Location authorizerJar = AppJarHelper.createDeploymentJar(locationFactory, InMemoryAuthorizer.class);
    cConf.set(Constants.Security.Authorization.EXTENSION_JAR_PATH, authorizerJar.toURI().getPath());
}

Example 9 with LocationFactory

use of org.apache.twill.filesystem.LocationFactory in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method setup.

@BeforeClass
public static void setup() throws Exception {
    cConf = CConfiguration.create();
    File dataDir = new File(TMP_FOLDER.newFolder(), "data");
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, dataDir.getAbsolutePath());
    final Injector injector = Guice.createInjector(new ConfigModule(cConf), new NonCustomLocationUnitTestModule().getModule(), new TransactionInMemoryModule(), new NamespaceClientRuntimeModule().getInMemoryModules(), new AuditModule().getInMemoryModules(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(OwnerStore.class).to(InMemoryOwnerStore.class).in(Scopes.SINGLETON);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    });
    locationFactory = injector.getInstance(LocationFactory.class);
    namespacedLocationFactory = injector.getInstance(NamespacedLocationFactory.class);
    txExecutorFactory = injector.getInstance(TransactionExecutorFactory.class);
    registryFactory = new DatasetDefinitionRegistryFactory() {

        @Override
        public DatasetDefinitionRegistry create() {
            DefaultDatasetDefinitionRegistry registry = new DefaultDatasetDefinitionRegistry();
            injector.injectMembers(registry);
            return registry;
        }
    };
    namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
    namespaceQueryAdmin = injector.getInstance(NamespaceQueryAdmin.class);
    ownerAdmin = injector.getInstance(OwnerAdmin.class);
    inMemoryAuditPublisher = injector.getInstance(InMemoryAuditPublisher.class);
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(NAMESPACE_ID).build());
}

Example 10 with LocationFactory

use of org.apache.twill.filesystem.LocationFactory in project cdap by caskdata.

the class FileMetadataTest method testFileMetadataReadWrite.

@Test
public void testFileMetadataReadWrite() throws Exception {
    DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
    DatasetManager datasetManager = new DefaultDatasetManager(datasetFramework, NamespaceId.SYSTEM, co.cask.cdap.common.service.RetryStrategies.noRetry(), null);
    Transactional transactional = Transactions.createTransactionalWithRetry(Transactions.createTransactional(new MultiThreadDatasetCache(new SystemDatasetInstantiator(datasetFramework), injector.getInstance(TransactionSystemClient.class), NamespaceId.SYSTEM, ImmutableMap.<String, String>of(), null, null)), RetryStrategies.retryOnConflict(20, 100));
    FileMetaDataWriter fileMetaDataWriter = new FileMetaDataWriter(datasetManager, transactional);
    LogPathIdentifier logPathIdentifier = new LogPathIdentifier(NamespaceId.DEFAULT.getNamespace(), "testApp", "testFlow");
    LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
    Location location = locationFactory.create(TMP_FOLDER.newFolder().getPath()).append("/logs");
    long currentTime = System.currentTimeMillis();
    for (int i = 10; i <= 100; i += 10) {
        // i is the event time
        fileMetaDataWriter.writeMetaData(logPathIdentifier, i, currentTime, location.append(Integer.toString(i)));
    }
    // for the timestamp 80, add new new log path id with different current time.
    fileMetaDataWriter.writeMetaData(logPathIdentifier, 80, currentTime + 1, location.append("81"));
    fileMetaDataWriter.writeMetaData(logPathIdentifier, 80, currentTime + 2, location.append("82"));
    // reader test
    FileMetaDataReader fileMetadataReader = injector.getInstance(FileMetaDataReader.class);
    Assert.assertEquals(12, fileMetadataReader.listFiles(logPathIdentifier, 0, 100).size());
    Assert.assertEquals(5, fileMetadataReader.listFiles(logPathIdentifier, 20, 50).size());
    Assert.assertEquals(2, fileMetadataReader.listFiles(logPathIdentifier, 100, 150).size());
    // should include the latest file with event start time 80.
    List<LogLocation> locationList = fileMetadataReader.listFiles(logPathIdentifier, 81, 85);
    Assert.assertEquals(1, locationList.size());
    Assert.assertEquals(80, locationList.get(0).getEventTimeMs());
    Assert.assertEquals(location.append("82"), locationList.get(0).getLocation());
    Assert.assertEquals(1, fileMetadataReader.listFiles(logPathIdentifier, 150, 1000).size());
}