Examples with SystemStreamPartitionMetadata org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata used on opensource projects

Example 1 with SystemStreamPartitionMetadata

use of org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata in project samza by apache.

the class EventHubSystemAdmin method getPartitionMetadata.

private Map<Partition, SystemStreamPartitionMetadata> getPartitionMetadata(String streamName, String[] partitionIds) {
    EventHubClientManager eventHubClientManager = getOrCreateStreamEventHubClient(streamName);
    Map<Partition, SystemStreamPartitionMetadata> sspMetadataMap = new HashMap<>();
    List<CompletableFuture<PartitionRuntimeInformation>> futureList = new ArrayList<>();
    for (String partition : partitionIds) {
        CompletableFuture<PartitionRuntimeInformation> partitionRuntimeInfo = eventHubClientManager.getEventHubClient().getPartitionRuntimeInformation(partition);
        futureList.add(partitionRuntimeInfo);
        partitionRuntimeInfo.thenAccept(ehPartitionInfo -> {
            LOG.info(printPartitionRuntimeInfo(ehPartitionInfo));
            // Set offsets
            String startingOffset = EventHubSystemConsumer.START_OF_STREAM;
            String newestOffset = ehPartitionInfo.getLastEnqueuedOffset();
            String upcomingOffset = EventHubSystemConsumer.END_OF_STREAM;
            SystemStreamPartitionMetadata sspMetadata = new SystemStreamPartitionMetadata(startingOffset, newestOffset, upcomingOffset);
            sspMetadataMap.put(new Partition(Integer.parseInt(partition)), sspMetadata);
        });
    }
    CompletableFuture<Void> futureGroup = CompletableFuture.allOf(futureList.toArray(new CompletableFuture[futureList.size()]));
    long timeoutMs = eventHubConfig.getRuntimeInfoWaitTimeMS(systemName);
    try {
        futureGroup.get(timeoutMs, TimeUnit.MILLISECONDS);
    } catch (InterruptedException | ExecutionException | TimeoutException e) {
        String msg = String.format("Error while fetching EventHubPartitionRuntimeInfo for System:%s, Stream:%s", systemName, streamName);
        LOG.error(msg, e);
        throw new SamzaException(msg, e);
    }
    return sspMetadataMap;
}

Example 2 with SystemStreamPartitionMetadata

use of org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata in project samza by apache.

the class TestKafkaCheckpointManager method testStart.

@Test
public void testStart() {
    setupSystemFactory(config());
    String oldestOffset = "1";
    String newestOffset = "2";
    SystemStreamMetadata checkpointTopicMetadata = new SystemStreamMetadata(CHECKPOINT_TOPIC, ImmutableMap.of(new Partition(0), new SystemStreamPartitionMetadata(oldestOffset, newestOffset, Integer.toString(Integer.parseInt(newestOffset) + 1))));
    when(this.systemAdmin.getSystemStreamMetadata(Collections.singleton(CHECKPOINT_TOPIC))).thenReturn(ImmutableMap.of(CHECKPOINT_TOPIC, checkpointTopicMetadata));
    KafkaCheckpointManager checkpointManager = buildKafkaCheckpointManager(true, config());
    checkpointManager.start();
    verify(this.systemProducer).start();
    verify(this.systemAdmin).start();
    verify(this.systemConsumer).register(CHECKPOINT_SSP, oldestOffset);
    verify(this.systemConsumer).start();
}

Example 3 with SystemStreamPartitionMetadata

use of org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata in project samza by apache.

the class TestTransactionalStateTaskBackupManager method testGetNewestOffsetsThrowsIfNullSSPMetadata.

@Test(expected = SamzaException.class)
public void testGetNewestOffsetsThrowsIfNullSSPMetadata() {
    // empty topic == null newest offset
    ContainerStorageManager csm = mock(ContainerStorageManager.class);
    KafkaTransactionalStateTaskBackupManager tsm = buildTSM(csm, mock(Partition.class), new StorageManagerUtil());
    TaskName taskName = mock(TaskName.class);
    String changelogSystemName = "systemName";
    String storeName = "storeName";
    String changelogStreamName = "changelogName";
    String newestChangelogSSPOffset = null;
    SystemStream changelogSystemStream = new SystemStream(changelogSystemName, changelogStreamName);
    Partition changelogPartition = new Partition(0);
    SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition);
    java.util.Map<String, SystemStream> storeChangelogs = new HashMap<>();
    storeChangelogs.put(storeName, changelogSystemStream);
    SystemAdmins systemAdmins = mock(SystemAdmins.class);
    SystemAdmin systemAdmin = mock(SystemAdmin.class);
    SystemStreamPartitionMetadata metadata = mock(SystemStreamPartitionMetadata.class);
    when(metadata.getNewestOffset()).thenReturn(newestChangelogSSPOffset);
    when(systemAdmins.getSystemAdmin(changelogSystemName)).thenReturn(systemAdmin);
    java.util.Map metadataMap = new HashMap() {

        {
            put(changelogSSP, null);
        }
    };
    when(systemAdmin.getSSPMetadata(eq(ImmutableSet.of(changelogSSP)))).thenReturn(metadataMap);
    // invoke the method
    java.util.Map<String, String> offsets = tsm.getNewestChangelogSSPOffsets(taskName, storeChangelogs, changelogPartition, systemAdmins);
    // verify results
    fail("Should have thrown an exception if admin returned null metadata for changelog SSP");
}

Example 4 with SystemStreamPartitionMetadata

use of org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata in project samza by apache.

the class TestTransactionalStateTaskBackupManager method testGetNewestOffsetsThrowsIfNullMetadata.

@Test(expected = SamzaException.class)
public void testGetNewestOffsetsThrowsIfNullMetadata() {
    // empty topic == null newest offset
    ContainerStorageManager csm = mock(ContainerStorageManager.class);
    KafkaTransactionalStateTaskBackupManager tsm = buildTSM(csm, mock(Partition.class), new StorageManagerUtil());
    TaskName taskName = mock(TaskName.class);
    String changelogSystemName = "systemName";
    String storeName = "storeName";
    String changelogStreamName = "changelogName";
    String newestChangelogSSPOffset = null;
    SystemStream changelogSystemStream = new SystemStream(changelogSystemName, changelogStreamName);
    Partition changelogPartition = new Partition(0);
    SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition);
    java.util.Map<String, SystemStream> storeChangelogs = new HashMap<>();
    storeChangelogs.put(storeName, changelogSystemStream);
    SystemAdmins systemAdmins = mock(SystemAdmins.class);
    SystemAdmin systemAdmin = mock(SystemAdmin.class);
    SystemStreamPartitionMetadata metadata = mock(SystemStreamPartitionMetadata.class);
    when(metadata.getNewestOffset()).thenReturn(newestChangelogSSPOffset);
    when(systemAdmins.getSystemAdmin(changelogSystemName)).thenReturn(systemAdmin);
    when(systemAdmin.getSSPMetadata(eq(ImmutableSet.of(changelogSSP)))).thenReturn(null);
    // invoke the method
    java.util.Map<String, String> offsets = tsm.getNewestChangelogSSPOffsets(taskName, storeChangelogs, changelogPartition, systemAdmins);
    // verify results
    fail("Should have thrown an exception if admin didn't return any metadata");
}

Example 5 with SystemStreamPartitionMetadata

use of org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata in project samza by apache.

the class TestTransactionalStateTaskBackupManager method testGetNewestOffsetsReturnsNoneForEmptyTopic.

@Test
public void testGetNewestOffsetsReturnsNoneForEmptyTopic() {
    // empty topic == null newest offset
    ContainerStorageManager csm = mock(ContainerStorageManager.class);
    KafkaTransactionalStateTaskBackupManager tsm = buildTSM(csm, mock(Partition.class), new StorageManagerUtil());
    TaskName taskName = mock(TaskName.class);
    String changelogSystemName = "systemName";
    String storeName = "storeName";
    String changelogStreamName = "changelogName";
    String newestChangelogSSPOffset = null;
    SystemStream changelogSystemStream = new SystemStream(changelogSystemName, changelogStreamName);
    Partition changelogPartition = new Partition(0);
    SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition);
    java.util.Map<String, SystemStream> storeChangelogs = new HashMap<String, SystemStream>();
    storeChangelogs.put(storeName, changelogSystemStream);
    SystemAdmins systemAdmins = mock(SystemAdmins.class);
    SystemAdmin systemAdmin = mock(SystemAdmin.class);
    SystemStreamPartitionMetadata metadata = mock(SystemStreamPartitionMetadata.class);
    when(metadata.getNewestOffset()).thenReturn(newestChangelogSSPOffset);
    when(systemAdmins.getSystemAdmin(changelogSystemName)).thenReturn(systemAdmin);
    when(systemAdmin.getSSPMetadata(eq(ImmutableSet.of(changelogSSP)))).thenReturn(ImmutableMap.of(changelogSSP, metadata));
    // invoke the method
    java.util.Map<String, String> stateCheckpointMarkerMap = tsm.getNewestChangelogSSPOffsets(taskName, storeChangelogs, changelogPartition, systemAdmins);
    // verify results
    assertEquals(1, stateCheckpointMarkerMap.size());
    KafkaStateCheckpointMarker kscm = KafkaStateCheckpointMarker.deserialize(stateCheckpointMarkerMap.get(storeName));
    assertEquals(changelogSSP, kscm.getChangelogSSP());
    assertNull(kscm.getChangelogOffset());
}