Examples with SystemConfig org.apache.samza.config.SystemConfig used on opensource projects

Example 1 with SystemConfig

use of org.apache.samza.config.SystemConfig in project samza by apache.

the class DiagnosticsUtil method createDiagnosticsStream.

public static void createDiagnosticsStream(Config config) {
    if (!new JobConfig(config).getDiagnosticsEnabled()) {
        return;
    }
    // if diagnostics is enabled, create diagnostics stream if it doesnt exist
    String diagnosticsSystemStreamName = new MetricsConfig(config).getMetricsSnapshotReporterStream(MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS).orElseThrow(() -> new ConfigException("Missing required config: " + String.format(MetricsConfig.METRICS_SNAPSHOT_REPORTER_STREAM, MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS)));
    SystemStream diagnosticsSystemStream = StreamUtil.getSystemStreamFromNames(diagnosticsSystemStreamName);
    SystemConfig systemConfig = new SystemConfig(config);
    SystemAdmin diagnosticsSysAdmin = systemConfig.getSystemFactories().get(diagnosticsSystemStream.getSystem()).getAdmin(diagnosticsSystemStream.getSystem(), config, DiagnosticsUtil.class.getSimpleName());
    StreamSpec diagnosticsStreamSpec = new StreamSpec(DIAGNOSTICS_STREAM_ID, diagnosticsSystemStream.getStream(), diagnosticsSystemStream.getSystem(), new StreamConfig(config).getStreamProperties(DIAGNOSTICS_STREAM_ID));
    log.info("Creating diagnostics stream {}", diagnosticsSystemStream.getStream());
    diagnosticsSysAdmin.start();
    if (diagnosticsSysAdmin.createStream(diagnosticsStreamSpec)) {
        log.info("Created diagnostics stream {}", diagnosticsSystemStream.getStream());
    } else {
        log.info("Diagnostics stream {} already exists", diagnosticsSystemStream.getStream());
    }
    diagnosticsSysAdmin.stop();
}

Example 2 with SystemConfig

use of org.apache.samza.config.SystemConfig in project samza by apache.

the class DiagnosticsUtil method buildDiagnosticsManager.

/**
 * Create a {@link DiagnosticsManager} for the given jobName, jobId, containerId, and execEnvContainerId, if
 * diagnostics is enabled.
 * @param executionEnvContainerId ID assigned to the container by the cluster manager (e.g. YARN)
 * @param samzaEpochId ID assigned to the job deployment attempt by the cluster manager
 */
public static Optional<DiagnosticsManager> buildDiagnosticsManager(String jobName, String jobId, JobModel jobModel, String containerId, Optional<String> executionEnvContainerId, Optional<String> samzaEpochId, Config config) {
    JobConfig jobConfig = new JobConfig(config);
    MetricsConfig metricsConfig = new MetricsConfig(config);
    Optional<DiagnosticsManager> diagnosticsManagerOptional = Optional.empty();
    if (jobConfig.getDiagnosticsEnabled()) {
        ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
        int containerMemoryMb = clusterManagerConfig.getContainerMemoryMb();
        int containerNumCores = clusterManagerConfig.getNumCores();
        long maxHeapSizeBytes = Runtime.getRuntime().maxMemory();
        int containerThreadPoolSize = jobConfig.getThreadPoolSize();
        String taskClassVersion = Util.getTaskClassVersion(config);
        String samzaVersion = Util.getSamzaVersion();
        String hostName = Util.getLocalHost().getHostName();
        Optional<String> diagnosticsReporterStreamName = metricsConfig.getMetricsSnapshotReporterStream(MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS);
        if (!diagnosticsReporterStreamName.isPresent()) {
            throw new ConfigException("Missing required config: " + String.format(MetricsConfig.METRICS_SNAPSHOT_REPORTER_STREAM, MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS));
        }
        SystemStream diagnosticsSystemStream = StreamUtil.getSystemStreamFromNames(diagnosticsReporterStreamName.get());
        // Create a SystemProducer for DiagnosticsManager. This producer is used by the DiagnosticsManager
        // to write to the same stream as the MetricsSnapshotReporter called `diagnosticsreporter`.
        Optional<String> diagnosticsSystemFactoryName = new SystemConfig(config).getSystemFactory(diagnosticsSystemStream.getSystem());
        if (!diagnosticsSystemFactoryName.isPresent()) {
            throw new SamzaException("Missing factory in config for system " + diagnosticsSystemStream.getSystem());
        }
        SystemFactory systemFactory = ReflectionUtil.getObj(diagnosticsSystemFactoryName.get(), SystemFactory.class);
        SystemProducer systemProducer = systemFactory.getProducer(diagnosticsSystemStream.getSystem(), config, new MetricsRegistryMap(), DiagnosticsUtil.class.getSimpleName());
        DiagnosticsManager diagnosticsManager = new DiagnosticsManager(jobName, jobId, jobModel.getContainers(), containerMemoryMb, containerNumCores, new StorageConfig(config).getNumPersistentStores(), maxHeapSizeBytes, containerThreadPoolSize, containerId, executionEnvContainerId.orElse(""), samzaEpochId.orElse(""), taskClassVersion, samzaVersion, hostName, diagnosticsSystemStream, systemProducer, Duration.ofMillis(new TaskConfig(config).getShutdownMs()), jobConfig.getAutosizingEnabled(), config);
        diagnosticsManagerOptional = Optional.of(diagnosticsManager);
    }
    return diagnosticsManagerOptional;
}

Example 3 with SystemConfig

use of org.apache.samza.config.SystemConfig in project samza by apache.

the class ChangelogStreamManager method createChangelogStreams.

/**
 * Creates and validates the changelog streams of a samza job.
 *
 * @param config the configuration with changelog info.
 * @param maxChangeLogStreamPartitions the maximum number of changelog stream partitions to create.
 */
public static void createChangelogStreams(Config config, int maxChangeLogStreamPartitions) {
    // Get changelog store config
    StorageConfig storageConfig = new StorageConfig(config);
    ImmutableMap.Builder<String, SystemStream> storeNameSystemStreamMapBuilder = new ImmutableMap.Builder<>();
    storageConfig.getStoreNames().forEach(storeName -> {
        Optional<String> changelogStream = storageConfig.getChangelogStream(storeName);
        if (changelogStream.isPresent() && StringUtils.isNotBlank(changelogStream.get())) {
            storeNameSystemStreamMapBuilder.put(storeName, StreamUtil.getSystemStreamFromNames(changelogStream.get()));
        }
    });
    Map<String, SystemStream> storeNameSystemStreamMapping = storeNameSystemStreamMapBuilder.build();
    // Get SystemAdmin for changelog store's system and attempt to create the stream
    SystemConfig systemConfig = new SystemConfig(config);
    storeNameSystemStreamMapping.forEach((storeName, systemStream) -> {
        // Load system admin for this system.
        SystemAdmin systemAdmin = systemConfig.getSystemFactories().get(systemStream.getSystem()).getAdmin(systemStream.getSystem(), config, ChangelogStreamManager.class.getSimpleName());
        if (systemAdmin == null) {
            throw new SamzaException(String.format("Error creating changelog. Changelog on store %s uses system %s, which is missing from the configuration.", storeName, systemStream.getSystem()));
        }
        StreamSpec changelogSpec = StreamSpec.createChangeLogStreamSpec(systemStream.getStream(), systemStream.getSystem(), maxChangeLogStreamPartitions);
        systemAdmin.start();
        if (systemAdmin.createStream(changelogSpec)) {
            LOG.info(String.format("created changelog stream %s.", systemStream.getStream()));
        } else {
            LOG.info(String.format("changelog stream %s already exists.", systemStream.getStream()));
        }
        systemAdmin.validateStream(changelogSpec);
        if (storageConfig.getAccessLogEnabled(storeName)) {
            String accesslogStream = storageConfig.getAccessLogStream(systemStream.getStream());
            StreamSpec accesslogSpec = new StreamSpec(accesslogStream, accesslogStream, systemStream.getSystem(), maxChangeLogStreamPartitions);
            systemAdmin.createStream(accesslogSpec);
            systemAdmin.validateStream(accesslogSpec);
        }
        systemAdmin.stop();
    });
}

Example 4 with SystemConfig

use of org.apache.samza.config.SystemConfig in project samza by apache.

the class StorageRecovery method getContainerStorageManagers.

/**
 * create one TaskStorageManager for each task. Add all of them to the
 * List<TaskStorageManager>
 */
@SuppressWarnings("rawtypes")
private void getContainerStorageManagers() {
    Set<String> factoryClasses = new StorageConfig(jobConfig).getRestoreFactories();
    Map<String, StateBackendFactory> stateBackendFactories = factoryClasses.stream().collect(Collectors.toMap(factoryClass -> factoryClass, factoryClass -> ReflectionUtil.getObj(factoryClass, StateBackendFactory.class)));
    Clock clock = SystemClock.instance();
    StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 5000, clock);
    // don't worry about prefetching for this; looks like the tool doesn't flush to offset files anyways
    Map<String, SystemFactory> systemFactories = new SystemConfig(jobConfig).getSystemFactories();
    CheckpointManager checkpointManager = new TaskConfig(jobConfig).getCheckpointManager(new MetricsRegistryMap()).orElse(null);
    for (ContainerModel containerModel : containers.values()) {
        ContainerContext containerContext = new ContainerContextImpl(containerModel, new MetricsRegistryMap());
        ContainerStorageManager containerStorageManager = new ContainerStorageManager(checkpointManager, containerModel, streamMetadataCache, systemAdmins, changeLogSystemStreams, new HashMap<>(), storageEngineFactories, systemFactories, this.getSerdes(), jobConfig, new HashMap<>(), new SamzaContainerMetrics(containerModel.getId(), new MetricsRegistryMap(), ""), JobContextImpl.fromConfigWithDefaults(jobConfig, jobModel), containerContext, stateBackendFactories, new HashMap<>(), storeBaseDir, storeBaseDir, null, new SystemClock());
        this.containerStorageManagers.put(containerModel.getId(), containerStorageManager);
    }
}

Example 5 with SystemConfig

use of org.apache.samza.config.SystemConfig in project samza by apache.

the class SystemConsumerWithSamzaBench method start.

public void start() throws IOException, InterruptedException {
    super.start();
    MessageConsumer consumeFn = new MessageConsumer();
    StreamApplication app = appDesc -> {
        String systemFactoryName = new SystemConfig(config).getSystemFactory(systemName).get();
        GenericSystemDescriptor sd = new GenericSystemDescriptor(systemName, systemFactoryName);
        GenericInputDescriptor<Object> isd = sd.getInputDescriptor(streamId, new NoOpSerde<>());
        MessageStream<Object> stream = appDesc.getInputStream(isd);
        stream.map(consumeFn);
    };
    ApplicationRunner runner = ApplicationRunners.getApplicationRunner(app, new MapConfig());
    runner.run();
    while (consumeFn.getEventsConsumed() < totalEvents) {
        Thread.sleep(10);
    }
    Instant endTime = Instant.now();
    runner.kill();
    System.out.println("\n*******************");
    System.out.println(String.format("Started at %s Ending at %s ", consumeFn.startTime, endTime));
    System.out.println(String.format("Event Rate is %s Messages/Sec ", consumeFn.getEventsConsumed() * 1000 / Duration.between(consumeFn.startTime, Instant.now()).toMillis()));
    System.out.println("Event Rate is " + consumeFn.getEventsConsumed() * 1000 / Duration.between(consumeFn.startTime, endTime).toMillis());
    System.out.println("*******************\n");
    System.exit(0);
}