Examples with SystemStreamMetadata org.apache.samza.system.SystemStreamMetadata used on opensource projects

Example 6 with SystemStreamMetadata

use of org.apache.samza.system.SystemStreamMetadata in project samza by apache.

the class HdfsSystemAdmin method getSystemStreamMetadata.

/**
   *
   * Fetch metadata from hdfs system for a set of streams. This has the potential side effect
   * to persist partition description to the staging directory on hdfs if staging directory
   * is not empty. See getStagingDirectory on {@link HdfsConfig}
   *
   * @param streamNames
   *          The streams to to fetch metadata for.
   * @return A map from stream name to SystemStreamMetadata for each stream
   *         requested in the parameter set.
   */
@Override
public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) {
    Map<String, SystemStreamMetadata> systemStreamMetadataMap = new HashMap<>();
    streamNames.forEach(streamName -> {
        systemStreamMetadataMap.put(streamName, new SystemStreamMetadata(streamName, directoryPartitioner.getPartitionMetadataMap(streamName, obtainPartitionDescriptorMap(stagingDirectory, streamName))));
        if (!partitionDescriptorExists(streamName)) {
            persistPartitionDescriptor(streamName, directoryPartitioner.getPartitionDescriptor(streamName));
        }
    });
    return systemStreamMetadataMap;
}

Example 7 with SystemStreamMetadata

use of org.apache.samza.system.SystemStreamMetadata in project samza by apache.

the class TestHdfsSystemConsumer method testEmptyStagingDirectory.

/*
   * Ensure that empty staging directory will not break system admin,
   * but should fail system consumer
   */
@Test
public void testEmptyStagingDirectory() throws Exception {
    Map<String, String> configMap = new HashMap<>();
    configMap.put(String.format(HdfsConfig.CONSUMER_PARTITIONER_WHITELIST(), SYSTEM_NAME), ".*avro");
    Config config = new MapConfig(configMap);
    HdfsSystemFactory systemFactory = new HdfsSystemFactory();
    // create admin and do partitioning
    HdfsSystemAdmin systemAdmin = systemFactory.getAdmin(SYSTEM_NAME, config);
    String stream = WORKING_DIRECTORY;
    Set<String> streamNames = new HashSet<>();
    streamNames.add(stream);
    generateAvroDataFiles();
    Map<String, SystemStreamMetadata> streamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames);
    SystemStreamMetadata systemStreamMetadata = streamMetadataMap.get(stream);
    Assert.assertEquals(NUM_FILES, systemStreamMetadata.getSystemStreamPartitionMetadata().size());
    // create consumer and read from files
    HdfsSystemConsumer systemConsumer = systemFactory.getConsumer(SYSTEM_NAME, config, new NoOpMetricsRegistry());
    Partition partition = new Partition(0);
    SystemStreamPartition ssp = new SystemStreamPartition(SYSTEM_NAME, stream, partition);
    try {
        systemConsumer.register(ssp, "0");
        Assert.fail("Empty staging directory should fail system consumer");
    } catch (UncheckedExecutionException e) {
        Assert.assertTrue(e.getCause() instanceof SamzaException);
    }
}

Example 8 with SystemStreamMetadata

use of org.apache.samza.system.SystemStreamMetadata in project samza by apache.

the class StreamPartitionCountMonitor method updatePartitionCountMetric.

/**
   * Fetches the current partition count for each system stream from the cache, compares the current count to the
   * original count and updates the metric for that system stream with the delta.
   */
void updatePartitionCountMetric() {
    try {
        Map<SystemStream, SystemStreamMetadata> currentMetadata = getMetadata(streamsToMonitor, metadataCache);
        for (Map.Entry<SystemStream, SystemStreamMetadata> metadataEntry : initialMetadata.entrySet()) {
            SystemStream systemStream = metadataEntry.getKey();
            SystemStreamMetadata metadata = metadataEntry.getValue();
            int currentPartitionCount = currentMetadata.get(systemStream).getSystemStreamPartitionMetadata().keySet().size();
            int prevPartitionCount = metadata.getSystemStreamPartitionMetadata().keySet().size();
            Gauge gauge = gauges.get(systemStream);
            gauge.set(currentPartitionCount - prevPartitionCount);
        }
    } catch (Exception e) {
        log.error("Exception while updating partition count metric.", e);
    }
}

Example 9 with SystemStreamMetadata

use of org.apache.samza.system.SystemStreamMetadata in project samza by apache.

the class StreamManager method getStreamPartitionCounts.

Map<String, Integer> getStreamPartitionCounts(String systemName, Set<String> streamNames) {
    Map<String, Integer> streamToPartitionCount = new HashMap<>();
    SystemAdmin systemAdmin = sysAdmins.get(systemName);
    if (systemAdmin == null) {
        throw new SamzaException(String.format("System %s does not exist.", systemName));
    }
    // retrieve the metadata for the streams in this system
    Map<String, SystemStreamMetadata> streamToMetadata = systemAdmin.getSystemStreamMetadata(streamNames);
    // set the partitions of a stream to its StreamEdge
    streamToMetadata.forEach((stream, data) -> streamToPartitionCount.put(stream, data.getSystemStreamPartitionMetadata().size()));
    return streamToPartitionCount;
}