Examples with MetadataStore org.apache.samza.metadatastore.MetadataStore used on opensource projects

Example 1 with MetadataStore

use of org.apache.samza.metadatastore.MetadataStore in project samza by apache.

the class TestLocalApplicationRunner method testCreateCoordinatorStreamWithoutCoordinatorFactory.

/**
 * Underlying coordinator stream should not be created if not using CoordinatorStreamMetadataStoreFactory
 */
@Test
public void testCreateCoordinatorStreamWithoutCoordinatorFactory() throws Exception {
    SystemAdmins systemAdmins = mock(SystemAdmins.class);
    PowerMockito.whenNew(SystemAdmins.class).withAnyArguments().thenReturn(systemAdmins);
    LocalApplicationRunner localApplicationRunner = spy(new LocalApplicationRunner(mockApp, config, new InMemoryMetadataStoreFactory()));
    doReturn(false).when(localApplicationRunner).createUnderlyingCoordinatorStream(eq(config));
    MetadataStore coordinatorStreamStore = localApplicationRunner.createCoordinatorStreamStore(config);
    assertTrue(coordinatorStreamStore instanceof InMemoryMetadataStore);
    // creating underlying coordinator stream should not be called for other coordinator stream metadata store types.
    verify(localApplicationRunner, never()).createUnderlyingCoordinatorStream(eq(config));
}

Example 2 with MetadataStore

use of org.apache.samza.metadatastore.MetadataStore in project samza by apache.

the class LocalApplicationRunner method createCoordinatorStreamStore.

@VisibleForTesting
MetadataStore createCoordinatorStreamStore(Config config) {
    if (metadataStoreFactory.isPresent()) {
        // TODO: Add missing metadata store abstraction for creating the underlying store to address SAMZA-2182
        if (metadataStoreFactory.get() instanceof CoordinatorStreamMetadataStoreFactory) {
            if (createUnderlyingCoordinatorStream(config)) {
                MetadataStore coordinatorStreamStore = metadataStoreFactory.get().getMetadataStore("NoOp", config, new MetricsRegistryMap());
                LOG.info("Created coordinator stream store of type: {}", coordinatorStreamStore.getClass().getSimpleName());
                return coordinatorStreamStore;
            }
        } else {
            MetadataStore otherMetadataStore = metadataStoreFactory.get().getMetadataStore("NoOp", config, new MetricsRegistryMap());
            LOG.info("Created alternative coordinator stream store of type: {}", otherMetadataStore.getClass().getSimpleName());
            return otherMetadataStore;
        }
    }
    LOG.warn("No coordinator stream store created.");
    return null;
}

Example 3 with MetadataStore

use of org.apache.samza.metadatastore.MetadataStore in project samza by apache.

the class LocalJobPlanner method checkAndCreateStreams.

private void checkAndCreateStreams(String lockId, List<StreamSpec> intStreams, StreamManager streamManager) throws TimeoutException {
    MetadataStore metadataStore = getMetadataStore();
    DistributedLock distributedLock = coordinationUtils.getLock(lockId);
    if (distributedLock == null || metadataStore == null) {
        LOG.warn("Processor {} failed to create utils. Each processor will attempt to create streams.", processorId);
        // each application process will try creating the streams, which requires stream creation to be idempotent
        streamManager.createStreams(intStreams);
        return;
    }
    // Start timer for timeout
    long startTime = System.currentTimeMillis();
    long lockTimeout = TimeUnit.MILLISECONDS.convert(CoordinationConstants.LOCK_TIMEOUT_MS, TimeUnit.MILLISECONDS);
    // does not die of timeout exception and comes back and checks for state and proceeds
    while ((System.currentTimeMillis() - startTime) < lockTimeout) {
        if (metadataStore.get(String.format(STREAM_CREATED_STATE_KEY, lockId)) != null) {
            LOG.info("Processor {} found streams created state data. They must've been created by another processor.", processorId);
            break;
        }
        try {
            if (distributedLock.lock(Duration.ofMillis(10000))) {
                LOG.info("lock acquired for streams creation by Processor " + processorId);
                streamManager.createStreams(intStreams);
                String streamCreatedMessage = "Streams created by processor " + processorId;
                metadataStore.put(String.format(STREAM_CREATED_STATE_KEY, lockId), streamCreatedMessage.getBytes("UTF-8"));
                metadataStore.flush();
                distributedLock.unlock();
                break;
            } else {
                LOG.info("Processor {} failed to get the lock for stream initialization. Will try again until time out", processorId);
            }
        } catch (UnsupportedEncodingException e) {
            String msg = String.format("Processor {} failed to encode string for stream initialization", processorId);
            throw new SamzaException(msg, e);
        }
    }
    if ((System.currentTimeMillis() - startTime) >= lockTimeout) {
        throw new TimeoutException(String.format("Processor {} failed to get the lock for stream initialization within {} milliseconds.", processorId, CoordinationConstants.LOCK_TIMEOUT_MS));
    }
}

Example 4 with MetadataStore

use of org.apache.samza.metadatastore.MetadataStore in project samza by apache.

the class LocalApplicationRunner method initializeRunId.

private void initializeRunId() {
    if (!isAppModeBatch) {
        LOG.info("Not BATCH mode and hence not generating run id");
        return;
    }
    if (!coordinationUtils.isPresent()) {
        LOG.warn("Coordination utils not present. Aborting run id generation. Will continue execution without a run id.");
        return;
    }
    try {
        MetadataStore metadataStore = getMetadataStoreForRunID();
        runIdGenerator = Optional.of(new RunIdGenerator(coordinationUtils.get(), metadataStore));
        runId = runIdGenerator.flatMap(RunIdGenerator::getRunId);
    } catch (Exception e) {
        LOG.warn("Failed to generate run id. Will continue execution without a run id.", e);
    }
}

Example 5 with MetadataStore

use of org.apache.samza.metadatastore.MetadataStore in project samza by apache.

the class LocalApplicationRunner method run.

@Override
public void run(ExternalContext externalContext) {
    initializeRunId();
    LocalJobPlanner planner = getPlanner();
    try {
        List<JobConfig> jobConfigs = planner.prepareJobs();
        // create the StreamProcessors
        if (jobConfigs.isEmpty()) {
            throw new SamzaException("No jobs to run.");
        }
        jobConfigs.forEach(jobConfig -> {
            LOG.debug("Starting job {} StreamProcessor with config {}", jobConfig.getName(), jobConfig);
            MetadataStore coordinatorStreamStore = createCoordinatorStreamStore(jobConfig);
            if (coordinatorStreamStore != null) {
                coordinatorStreamStore.init();
            }
            StreamProcessor processor = createStreamProcessor(jobConfig, appDesc, sp -> new LocalStreamProcessorLifecycleListener(sp, jobConfig), Optional.ofNullable(externalContext), coordinatorStreamStore);
            processors.add(Pair.of(processor, coordinatorStreamStore));
        });
        numProcessorsToStart.set(processors.size());
        // start the StreamProcessors
        processors.forEach(sp -> sp.getLeft().start());
    } catch (Throwable throwable) {
        cleanup();
        appStatus = ApplicationStatus.unsuccessfulFinish(throwable);
        shutdownLatch.countDown();
        throw new SamzaException(String.format("Failed to start application: %s", new ApplicationConfig(appDesc.getConfig()).getGlobalAppId()), throwable);
    }
}