Search in sources :

Example 1 with SingleThreadDatasetCache

use of co.cask.cdap.data2.dataset2.SingleThreadDatasetCache in project cdap by caskdata.

the class WorkerProgramRunnerTest method beforeClass.

@BeforeClass
public static void beforeClass() throws IOException {
    // we are only gonna do long-running transactions here. Set the tx timeout to a ridiculously low value.
    // that will test that the long-running transactions actually bypass that timeout.
    CConfiguration conf = CConfiguration.create();
    conf.setInt(TxConstants.Manager.CFG_TX_TIMEOUT, 1);
    conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 2);
    Injector injector = AppFabricTestHelper.getInjector(conf);
    txService = injector.getInstance(TransactionManager.class);
    txExecutorFactory = injector.getInstance(TransactionExecutorFactory.class);
    dsFramework = injector.getInstance(DatasetFramework.class);
    datasetCache = new SingleThreadDatasetCache(new SystemDatasetInstantiator(dsFramework, WorkerProgramRunnerTest.class.getClassLoader(), null), injector.getInstance(TransactionSystemClient.class), NamespaceId.DEFAULT, DatasetDefinition.NO_ARGUMENTS, null, null);
    metricStore = injector.getInstance(MetricStore.class);
    txService.startAndWait();
}
Also used : DatasetFramework(co.cask.cdap.data2.dataset2.DatasetFramework) MetricStore(co.cask.cdap.api.metrics.MetricStore) Injector(com.google.inject.Injector) TransactionManager(org.apache.tephra.TransactionManager) SystemDatasetInstantiator(co.cask.cdap.data.dataset.SystemDatasetInstantiator) SingleThreadDatasetCache(co.cask.cdap.data2.dataset2.SingleThreadDatasetCache) CConfiguration(co.cask.cdap.common.conf.CConfiguration) TransactionExecutorFactory(co.cask.cdap.data2.transaction.TransactionExecutorFactory) BeforeClass(org.junit.BeforeClass)

Example 2 with SingleThreadDatasetCache

use of co.cask.cdap.data2.dataset2.SingleThreadDatasetCache in project cdap by caskdata.

the class MapReduceRunnerTestBase method beforeClass.

@BeforeClass
public static void beforeClass() throws Exception {
    CConfiguration conf = CConfiguration.create();
    // allow subclasses to override the following two parameters
    Integer txTimeout = Integer.getInteger(TxConstants.Manager.CFG_TX_TIMEOUT);
    if (txTimeout != null) {
        conf.setInt(TxConstants.Manager.CFG_TX_TIMEOUT, txTimeout);
    }
    Integer txCleanupInterval = Integer.getInteger(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL);
    if (txCleanupInterval != null) {
        conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, txCleanupInterval);
    }
    injector = AppFabricTestHelper.getInjector(conf, new AbstractModule() {

        @Override
        protected void configure() {
            bind(StreamFileWriterFactory.class).to(LocationStreamFileWriterFactory.class);
        }
    });
    txService = injector.getInstance(TransactionManager.class);
    txExecutorFactory = injector.getInstance(TransactionExecutorFactory.class);
    dsFramework = injector.getInstance(DatasetFramework.class);
    datasetCache = new SingleThreadDatasetCache(new SystemDatasetInstantiator(dsFramework, MapReduceRunnerTestBase.class.getClassLoader(), null), injector.getInstance(TransactionSystemClient.class), NamespaceId.DEFAULT, DatasetDefinition.NO_ARGUMENTS, null, null);
    metricStore = injector.getInstance(MetricStore.class);
    txService.startAndWait();
    streamHandler = injector.getInstance(StreamHandler.class);
    // Always create the default namespace
    injector.getInstance(NamespaceAdmin.class).create(NamespaceMeta.DEFAULT);
}
Also used : MetricStore(co.cask.cdap.api.metrics.MetricStore) NamespaceAdmin(co.cask.cdap.common.namespace.NamespaceAdmin) SingleThreadDatasetCache(co.cask.cdap.data2.dataset2.SingleThreadDatasetCache) CConfiguration(co.cask.cdap.common.conf.CConfiguration) AbstractModule(com.google.inject.AbstractModule) TransactionExecutorFactory(co.cask.cdap.data2.transaction.TransactionExecutorFactory) DatasetFramework(co.cask.cdap.data2.dataset2.DatasetFramework) StreamFileWriterFactory(co.cask.cdap.data.stream.StreamFileWriterFactory) LocationStreamFileWriterFactory(co.cask.cdap.data.runtime.LocationStreamFileWriterFactory) TransactionManager(org.apache.tephra.TransactionManager) SystemDatasetInstantiator(co.cask.cdap.data.dataset.SystemDatasetInstantiator) StreamHandler(co.cask.cdap.data.stream.service.StreamHandler) BeforeClass(org.junit.BeforeClass)

Example 3 with SingleThreadDatasetCache

use of co.cask.cdap.data2.dataset2.SingleThreadDatasetCache in project cdap by caskdata.

the class MultiConsumerTest method testMulti.

@Test
public void testMulti() throws Exception {
    // TODO: Fix this test case to really test with numGroups settings.
    final ApplicationWithPrograms app = AppFabricTestHelper.deployApplicationWithManager(MultiApp.class, TEMP_FOLDER_SUPPLIER);
    List<ProgramController> controllers = Lists.newArrayList();
    for (ProgramDescriptor programDescriptor : app.getPrograms()) {
        controllers.add(AppFabricTestHelper.submit(app, programDescriptor.getSpecification().getClassName(), new BasicArguments(), TEMP_FOLDER_SUPPLIER));
    }
    DatasetFramework datasetFramework = AppFabricTestHelper.getInjector().getInstance(DatasetFramework.class);
    DynamicDatasetCache datasetCache = new SingleThreadDatasetCache(new SystemDatasetInstantiator(datasetFramework, getClass().getClassLoader(), null), AppFabricTestHelper.getInjector().getInstance(TransactionSystemClient.class), NamespaceId.DEFAULT, DatasetDefinition.NO_ARGUMENTS, null, null);
    final KeyValueTable accumulated = datasetCache.getDataset("accumulated");
    TransactionExecutorFactory txExecutorFactory = AppFabricTestHelper.getInjector().getInstance(TransactionExecutorFactory.class);
    // Try to get accumulated result and verify it. Expect result appear in max of 60 seconds.
    int trial = 0;
    while (trial < 60) {
        try {
            Transactions.createTransactionExecutor(txExecutorFactory, accumulated).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    byte[] value = accumulated.read(MultiApp.KEY);
                    // Sum(1..100) * 3
                    Assert.assertEquals(((1 + 99) * 99 / 2) * 3, Longs.fromByteArray(value));
                }
            });
            break;
        } catch (TransactionFailureException e) {
            // No-op
            trial++;
            TimeUnit.SECONDS.sleep(1);
        }
    }
    Assert.assertTrue(trial < 60);
    for (ProgramController controller : controllers) {
        controller.stop().get();
    }
}
Also used : ProgramController(co.cask.cdap.app.runtime.ProgramController) DynamicDatasetCache(co.cask.cdap.data2.dataset2.DynamicDatasetCache) TransactionExecutor(org.apache.tephra.TransactionExecutor) SingleThreadDatasetCache(co.cask.cdap.data2.dataset2.SingleThreadDatasetCache) TransactionFailureException(org.apache.tephra.TransactionFailureException) IOException(java.io.IOException) TransactionExecutorFactory(org.apache.tephra.TransactionExecutorFactory) DatasetFramework(co.cask.cdap.data2.dataset2.DatasetFramework) TransactionSystemClient(org.apache.tephra.TransactionSystemClient) TransactionFailureException(org.apache.tephra.TransactionFailureException) ApplicationWithPrograms(co.cask.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms) SystemDatasetInstantiator(co.cask.cdap.data.dataset.SystemDatasetInstantiator) KeyValueTable(co.cask.cdap.api.dataset.lib.KeyValueTable) ProgramDescriptor(co.cask.cdap.app.program.ProgramDescriptor) BasicArguments(co.cask.cdap.internal.app.runtime.BasicArguments) Test(org.junit.Test)

Aggregations

SystemDatasetInstantiator (co.cask.cdap.data.dataset.SystemDatasetInstantiator)3 DatasetFramework (co.cask.cdap.data2.dataset2.DatasetFramework)3 SingleThreadDatasetCache (co.cask.cdap.data2.dataset2.SingleThreadDatasetCache)3 MetricStore (co.cask.cdap.api.metrics.MetricStore)2 CConfiguration (co.cask.cdap.common.conf.CConfiguration)2 TransactionExecutorFactory (co.cask.cdap.data2.transaction.TransactionExecutorFactory)2 TransactionManager (org.apache.tephra.TransactionManager)2 BeforeClass (org.junit.BeforeClass)2 KeyValueTable (co.cask.cdap.api.dataset.lib.KeyValueTable)1 ProgramDescriptor (co.cask.cdap.app.program.ProgramDescriptor)1 ProgramController (co.cask.cdap.app.runtime.ProgramController)1 NamespaceAdmin (co.cask.cdap.common.namespace.NamespaceAdmin)1 LocationStreamFileWriterFactory (co.cask.cdap.data.runtime.LocationStreamFileWriterFactory)1 StreamFileWriterFactory (co.cask.cdap.data.stream.StreamFileWriterFactory)1 StreamHandler (co.cask.cdap.data.stream.service.StreamHandler)1 DynamicDatasetCache (co.cask.cdap.data2.dataset2.DynamicDatasetCache)1 ApplicationWithPrograms (co.cask.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms)1 BasicArguments (co.cask.cdap.internal.app.runtime.BasicArguments)1 AbstractModule (com.google.inject.AbstractModule)1 Injector (com.google.inject.Injector)1