Examples with TransactionRunner io.cdap.cdap.spi.data.transaction.TransactionRunner used on opensource projects

Example 1 with TransactionRunner

use of io.cdap.cdap.spi.data.transaction.TransactionRunner in project cdap by caskdata.

the class AppMetadataStoreTest method testGetActiveRuns.

@Test
public void testGetActiveRuns() throws Exception {
    // write a run record for each state for two programs in two apps in two namespaces
    String app1 = "app1";
    String app2 = "app2";
    String program1 = "prog1";
    String program2 = "prog2";
    Collection<NamespaceId> namespaces = Arrays.asList(new NamespaceId("ns1"), new NamespaceId("ns2"));
    Collection<ApplicationId> apps = namespaces.stream().flatMap(ns -> Stream.of(ns.app(app1), ns.app(app2))).collect(Collectors.toList());
    Collection<ProgramId> programs = apps.stream().flatMap(app -> Stream.of(app.mr(program1), app.mr(program2))).collect(Collectors.toList());
    for (ProgramId programId : programs) {
        TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore store = AppMetadataStore.create(context);
            // one run in pending state
            ProgramRunId runId = programId.run(RunIds.generate());
            store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
            // one run in starting state
            runId = programId.run(RunIds.generate());
            store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
            store.recordProgramProvisioned(runId, 3, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramStart(runId, UUID.randomUUID().toString(), Collections.emptyMap(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            // one run in running state
            runId = programId.run(RunIds.generate());
            store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
            store.recordProgramProvisioned(runId, 3, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            String twillRunId = UUID.randomUUID().toString();
            store.recordProgramStart(runId, twillRunId, Collections.emptyMap(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramRunning(runId, System.currentTimeMillis(), twillRunId, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            // one in suspended state
            runId = programId.run(RunIds.generate());
            store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
            store.recordProgramProvisioned(runId, 3, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            twillRunId = UUID.randomUUID().toString();
            store.recordProgramStart(runId, twillRunId, Collections.emptyMap(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramRunning(runId, System.currentTimeMillis(), twillRunId, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramSuspend(runId, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), System.currentTimeMillis());
            // one run in stopping state
            runId = programId.run(RunIds.generate());
            store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
            store.recordProgramProvisioned(runId, 3, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            twillRunId = UUID.randomUUID().toString();
            store.recordProgramStart(runId, twillRunId, Collections.emptyMap(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramRunning(runId, System.currentTimeMillis(), twillRunId, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramStopping(runId, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), System.currentTimeMillis(), System.currentTimeMillis() + 1000);
            // one run in each stopped state
            for (ProgramRunStatus runStatus : ProgramRunStatus.values()) {
                if (!runStatus.isEndState()) {
                    continue;
                }
                runId = programId.run(RunIds.generate());
                store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
                store.recordProgramProvisioned(runId, 3, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
                twillRunId = UUID.randomUUID().toString();
                store.recordProgramStart(runId, twillRunId, Collections.emptyMap(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
                store.recordProgramStop(runId, System.currentTimeMillis(), runStatus, null, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            }
        });
    }
    Set<ProgramRunStatus> activeStates = new HashSet<>();
    activeStates.add(ProgramRunStatus.PENDING);
    activeStates.add(ProgramRunStatus.STARTING);
    activeStates.add(ProgramRunStatus.RUNNING);
    activeStates.add(ProgramRunStatus.SUSPENDED);
    activeStates.add(ProgramRunStatus.STOPPING);
    // test the instance level method and namespace level method
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore store = AppMetadataStore.create(context);
        Map<ProgramId, Set<ProgramRunStatus>> allExpected = new HashMap<>();
        Map<ProgramId, Set<ProgramRunStatus>> allActual = new HashMap<>();
        // check active runs per namespace
        for (NamespaceId namespace : namespaces) {
            Map<ProgramRunId, RunRecordDetail> activeRuns = store.getActiveRuns(namespace);
            // we expect 4 runs per program, with 4 programs in each namespace
            Map<ProgramId, Set<ProgramRunStatus>> expected = new HashMap<>();
            expected.put(namespace.app(app1).mr(program1), activeStates);
            expected.put(namespace.app(app1).mr(program2), activeStates);
            expected.put(namespace.app(app2).mr(program1), activeStates);
            expected.put(namespace.app(app2).mr(program2), activeStates);
            Map<ProgramId, Set<ProgramRunStatus>> actual = new HashMap<>();
            actual.put(namespace.app(app1).mr(program1), new HashSet<>());
            actual.put(namespace.app(app1).mr(program2), new HashSet<>());
            actual.put(namespace.app(app2).mr(program1), new HashSet<>());
            actual.put(namespace.app(app2).mr(program2), new HashSet<>());
            allActual.putAll(actual);
            for (Map.Entry<ProgramRunId, RunRecordDetail> activeRun : activeRuns.entrySet()) {
                ProgramId programId = activeRun.getKey().getParent();
                Assert.assertTrue("Unexpected program returned: " + programId, actual.containsKey(activeRun.getKey().getParent()));
                actual.get(programId).add(activeRun.getValue().getStatus());
            }
            Assert.assertEquals(expected, actual);
            allExpected.putAll(expected);
        }
        // test the instance level method
        for (Map.Entry<ProgramRunId, RunRecordDetail> activeRun : store.getActiveRuns(x -> true).entrySet()) {
            ProgramId programId = activeRun.getKey().getParent();
            Assert.assertTrue("Unexpected program returned: " + programId, allActual.containsKey(activeRun.getKey().getParent()));
            allActual.get(programId).add(activeRun.getValue().getStatus());
        }
        Assert.assertEquals(allExpected, allActual);
        // test the count-all method
        Assert.assertEquals(store.getActiveRuns(x -> true).size(), store.countActiveRuns(null));
        Assert.assertEquals(store.getActiveRuns(x -> true).size(), store.countActiveRuns(100));
        Assert.assertEquals(2, store.countActiveRuns(2));
    });
    // check active runs per app
    for (ApplicationId app : apps) {
        TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore store = AppMetadataStore.create(context);
            Map<ProgramRunId, RunRecordDetail> activeRuns = store.getActiveRuns(app);
            // we expect 3 runs per program, with 2 programs in each app
            Map<ProgramId, Set<ProgramRunStatus>> expected = new HashMap<>();
            expected.put(app.mr(program1), activeStates);
            expected.put(app.mr(program2), activeStates);
            Map<ProgramId, Set<ProgramRunStatus>> actual = new HashMap<>();
            actual.put(app.mr(program1), new HashSet<>());
            actual.put(app.mr(program2), new HashSet<>());
            for (Map.Entry<ProgramRunId, RunRecordDetail> activeRun : activeRuns.entrySet()) {
                ProgramId programId = activeRun.getKey().getParent();
                Assert.assertTrue("Unexpected program returned: " + programId, actual.containsKey(activeRun.getKey().getParent()));
                actual.get(programId).add(activeRun.getValue().getStatus());
            }
            Assert.assertEquals(expected, actual);
        });
    }
    // check active runs per program
    for (ProgramId program : programs) {
        TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore store = AppMetadataStore.create(context);
            Map<ProgramRunId, RunRecordDetail> activeRuns = store.getActiveRuns(program);
            Set<ProgramRunStatus> actual = new HashSet<>();
            for (Map.Entry<ProgramRunId, RunRecordDetail> activeRun : activeRuns.entrySet()) {
                Assert.assertEquals(program, activeRun.getKey().getParent());
                actual.add(activeRun.getValue().getStatus());
            }
            Assert.assertEquals(activeStates, actual);
        });
    }
}

Example 2 with TransactionRunner

use of io.cdap.cdap.spi.data.transaction.TransactionRunner in project cdap by caskdata.

the class SqlAppMetadataStoreTest method beforeClass.

@BeforeClass
public static void beforeClass() throws IOException, TableAlreadyExistsException {
    CConfiguration cConf = CConfiguration.create();
    pg = PostgresInstantiator.createAndStart(cConf, TEMP_FOLDER.newFolder());
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new LocalLocationModule(), new SystemDatasetRuntimeModule().getInMemoryModules(), new StorageModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class).in(Scopes.SINGLETON);
        }
    });
    transactionRunner = injector.getInstance(TransactionRunner.class);
    StoreDefinition.AppMetadataStore.create(injector.getInstance(StructuredTableAdmin.class));
}

Example 3 with TransactionRunner

use of io.cdap.cdap.spi.data.transaction.TransactionRunner in project cdap by caskdata.

the class MetadataSubscriberService method preProcess.

@Override
protected void preProcess() {
    if (didBackfill) {
        return;
    }
    if (backfillAttempts > 10) {
        LOG.info("Skipping attempt to back-fill plugin metadata after 10 failures.");
        return;
    }
    // Backfill plugin metadata
    backfillAttempts++;
    LOG.info("Starting back-fill process(attempt {}) for plugin metadata", backfillAttempts);
    boolean updateFailed = false;
    NamespaceStore namespaceStore = new DefaultNamespaceStore(this.transactionRunner);
    List<String> namespaces = namespaceStore.list().stream().map(NamespaceMeta::getName).collect(Collectors.toList());
    LOG.debug("Back-filling plugin metadata for {} namespaces", namespaces.size());
    for (String namespace : namespaces) {
        List<ApplicationMeta> apps = TransactionRunners.run(this.transactionRunner, context -> {
            AppMetadataStore appMetadataStore = AppMetadataStore.create(context);
            return appMetadataStore.getAllApplications(namespace);
        });
        LOG.debug("Back-filling plugin metadata for namespace '{}' with {} applications", namespace, apps.size());
        try {
            this.getPluginCounts(namespace, apps);
        } catch (IOException e) {
            updateFailed = true;
            LOG.warn("Failed to write plugin metadata updates for namespace '{}': {}", namespace, e);
        }
    }
    if (!updateFailed) {
        LOG.info("Successfully back-filled plugin metadata for {} namespaces.", namespaces.size());
        didBackfill = true;
        TransactionRunners.run(transactionRunner, (TxRunnable) context -> AppMetadataStore.create(context).persistSubscriberState(getTopicId().getTopic(), BACKFILL_SUBSCRIBER_NAME, "true"));
    }
}

Example 4 with TransactionRunner

use of io.cdap.cdap.spi.data.transaction.TransactionRunner in project cdap by caskdata.

the class DefaultStore method getRuns.

@Override
public List<ProgramHistory> getRuns(Collection<ProgramId> programs, ProgramRunStatus status, long startTime, long endTime, int limitPerProgram) {
    return TransactionRunners.run(transactionRunner, context -> {
        List<ProgramHistory> result = new ArrayList<>(programs.size());
        AppMetadataStore appMetadataStore = getAppMetadataStore(context);
        Set<ProgramId> existingPrograms = appMetadataStore.filterProgramsExistence(programs);
        for (ProgramId programId : programs) {
            if (!existingPrograms.contains(programId)) {
                result.add(new ProgramHistory(programId, Collections.emptyList(), new ProgramNotFoundException(programId)));
                continue;
            }
            List<RunRecord> runs = appMetadataStore.getRuns(programId, status, startTime, endTime, limitPerProgram, null).values().stream().map(record -> RunRecord.builder(record).build()).collect(Collectors.toList());
            result.add(new ProgramHistory(programId, runs, null));
        }
        return result;
    });
}

Example 5 with TransactionRunner

use of io.cdap.cdap.spi.data.transaction.TransactionRunner in project cdap by caskdata.

the class SqlProgramScheduleStoreDatasetTest method setup.

@BeforeClass
public static void setup() throws IOException, TableAlreadyExistsException {
    CConfiguration cConf = CConfiguration.create();
    // any plugin which requires transaction will be excluded
    cConf.set(Constants.REQUIREMENTS_DATASET_TYPE_EXCLUDE, Joiner.on(",").join(Table.TYPE, KeyValueTable.TYPE));
    pg = PostgresInstantiator.createAndStart(cConf, TEMP_FOLDER.newFolder());
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new LocalLocationModule(), new SystemDatasetRuntimeModule().getInMemoryModules(), new StorageModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class).in(Scopes.SINGLETON);
        }
    });
    transactionRunner = injector.getInstance(TransactionRunner.class);
    StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
}