Examples with TaskName org.apache.samza.container.TaskName used on opensource projects

Example 61 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestStartpointManager method testFanOutBasic.

@Test
public void testFanOutBasic() throws IOException {
    SystemStreamPartition sspBroadcast = new SystemStreamPartition("mockSystem1", "mockStream1", new Partition(2));
    SystemStreamPartition sspSingle = new SystemStreamPartition("mockSystem2", "mockStream2", new Partition(3));
    TaskName taskWithNonBroadcast = new TaskName("t1");
    List<TaskName> tasks = ImmutableList.of(new TaskName("t0"), taskWithNonBroadcast, new TaskName("t2"), new TaskName("t3"), new TaskName("t4"), new TaskName("t5"));
    Map<TaskName, Set<SystemStreamPartition>> taskToSSPs = tasks.stream().collect(Collectors.toMap(task -> task, task -> task.equals(taskWithNonBroadcast) ? ImmutableSet.of(sspBroadcast, sspSingle) : ImmutableSet.of(sspBroadcast)));
    StartpointSpecific startpoint42 = new StartpointSpecific("42");
    startpointManager.writeStartpoint(sspBroadcast, startpoint42);
    startpointManager.writeStartpoint(sspSingle, startpoint42);
    // startpoint42 should remap with key sspBroadcast to all tasks + sspBroadcast
    Map<TaskName, Map<SystemStreamPartition, Startpoint>> tasksFannedOutTo = startpointManager.fanOut(taskToSSPs);
    Assert.assertEquals(tasks.size(), tasksFannedOutTo.size());
    Assert.assertTrue(tasksFannedOutTo.keySet().containsAll(tasks));
    Assert.assertFalse("Should be deleted after fan out", startpointManager.readStartpoint(sspBroadcast).isPresent());
    Assert.assertFalse("Should be deleted after fan out", startpointManager.readStartpoint(sspSingle).isPresent());
    for (TaskName taskName : tasks) {
        Map<SystemStreamPartition, Startpoint> fanOutForTask = startpointManager.getFanOutForTask(taskName);
        if (taskName.equals(taskWithNonBroadcast)) {
            // Non-broadcast startpoint should be fanned out to only one task
            Assert.assertEquals("Should have broadcast and non-broadcast SSP", 2, fanOutForTask.size());
        } else {
            Assert.assertEquals("Should only have broadcast SSP", 1, fanOutForTask.size());
        }
        // Broadcast SSP should be on every task
        Startpoint startpointFromStore = fanOutForTask.get(sspBroadcast);
        Assert.assertEquals(StartpointSpecific.class, startpointFromStore.getClass());
        Assert.assertEquals(startpoint42.getSpecificOffset(), ((StartpointSpecific) startpointFromStore).getSpecificOffset());
        // startpoint mapped only to task "t1" for Non-broadcast SSP
        startpointFromStore = fanOutForTask.get(sspSingle);
        if (taskName.equals(taskWithNonBroadcast)) {
            Assert.assertEquals(StartpointSpecific.class, startpointFromStore.getClass());
            Assert.assertEquals(startpoint42.getSpecificOffset(), ((StartpointSpecific) startpointFromStore).getSpecificOffset());
        } else {
            Assert.assertNull("Should not have non-broadcast SSP", startpointFromStore);
        }
        startpointManager.removeFanOutForTask(taskName);
        Assert.assertTrue(startpointManager.getFanOutForTask(taskName).isEmpty());
    }
}

Example 62 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestStartpointManager method testDeleteAllStartpoints.

@Test
public void testDeleteAllStartpoints() throws IOException {
    SystemStreamPartition sspBroadcast = new SystemStreamPartition("mockSystem1", "mockStream1", new Partition(2));
    SystemStreamPartition sspSingle = new SystemStreamPartition("mockSystem2", "mockStream2", new Partition(3));
    TaskName taskWithNonBroadcast = new TaskName("t1");
    List<TaskName> tasks = ImmutableList.of(new TaskName("t0"), taskWithNonBroadcast, new TaskName("t2"), new TaskName("t3"), new TaskName("t4"), new TaskName("t5"));
    Map<TaskName, Set<SystemStreamPartition>> taskToSSPs = tasks.stream().collect(Collectors.toMap(task -> task, task -> task.equals(taskWithNonBroadcast) ? ImmutableSet.of(sspBroadcast, sspSingle) : ImmutableSet.of(sspBroadcast)));
    StartpointSpecific startpoint42 = new StartpointSpecific("42");
    startpointManager.writeStartpoint(sspBroadcast, startpoint42);
    startpointManager.writeStartpoint(sspSingle, startpoint42);
    // startpoint42 should remap with key sspBroadcast to all tasks + sspBroadcast
    Map<TaskName, Map<SystemStreamPartition, Startpoint>> tasksFannedOutTo = startpointManager.fanOut(taskToSSPs);
    Assert.assertEquals(tasks.size(), tasksFannedOutTo.size());
    Assert.assertTrue(tasksFannedOutTo.keySet().containsAll(tasks));
    Assert.assertFalse("Should be deleted after fan out", startpointManager.readStartpoint(sspBroadcast).isPresent());
    Assert.assertFalse("Should be deleted after fan out", startpointManager.readStartpoint(sspSingle).isPresent());
    // Re-populate startpoints after fan out
    startpointManager.writeStartpoint(sspBroadcast, startpoint42);
    startpointManager.writeStartpoint(sspSingle, startpoint42);
    Assert.assertEquals(2, startpointManager.getReadWriteStore().all().size());
    startpointManager.deleteAllStartpoints();
    Assert.assertEquals(0, startpointManager.getReadWriteStore().all().size());
    // Fan outs should be untouched
    Assert.assertEquals(tasks.size(), startpointManager.getFanOutStore().all().size());
}

Example 63 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestSamzaObjectMapper method setup.

@Before
public void setup() {
    Config config = new MapConfig(ImmutableMap.of("a", "b"));
    TaskName taskName = new TaskName("test");
    Set<SystemStreamPartition> ssps = ImmutableSet.of(new SystemStreamPartition("foo", "bar", new Partition(1)));
    TaskModel taskModel = new TaskModel(taskName, ssps, new Partition(2));
    Map<TaskName, TaskModel> tasks = ImmutableMap.of(taskName, taskModel);
    ContainerModel containerModel = new ContainerModel("1", tasks);
    Map<String, ContainerModel> containerMap = ImmutableMap.of("1", containerModel);
    this.jobModel = new JobModel(config, containerMap);
    this.samzaObjectMapper = SamzaObjectMapper.getObjectMapper();
}

Example 64 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestBlobStoreBackupManager method setup.

@Before
public void setup() throws Exception {
    when(clock.currentTimeMillis()).thenReturn(1234567L);
    // setup test local and remote snapshots
    indexBlobIdAndLocalRemoteSnapshotsPair = setupRemoteAndLocalSnapshots(true);
    // setup test store name and SCMs map
    testStoreNameAndSCMMap = setupTestStoreSCMMapAndStoreBackedFactoryConfig(indexBlobIdAndLocalRemoteSnapshotsPair);
    // setup: setup task backup manager with expected storeName->storageEngine map
    testStoreNameAndSCMMap.forEach((storeName, scm) -> storeStorageEngineMap.put(storeName, null));
    mapConfig.putAll(new MapConfig(ImmutableMap.of("job.name", jobName, "job.id", jobId)));
    Config config = new MapConfig(mapConfig);
    // Mock - return snapshot index for blob id from test blob store map
    ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
    when(blobStoreUtil.getSnapshotIndex(captor.capture(), any(Metadata.class))).then((Answer<CompletableFuture<SnapshotIndex>>) invocation -> {
        String blobId = invocation.getArgumentAt(0, String.class);
        return CompletableFuture.completedFuture(testBlobStore.get(blobId));
    });
    // doNothing().when(blobStoreManager).init();
    when(taskModel.getTaskName().getTaskName()).thenReturn(taskName);
    when(taskModel.getTaskMode()).thenReturn(TaskMode.Active);
    when(metricsRegistry.newCounter(anyString(), anyString())).thenReturn(counter);
    when(metricsRegistry.newGauge(anyString(), anyString(), anyLong())).thenReturn(longGauge);
    when(metricsRegistry.newGauge(anyString(), anyString(), any(AtomicLong.class))).thenReturn(atomicLongGauge);
    when(atomicLongGauge.getValue()).thenReturn(new AtomicLong());
    when(metricsRegistry.newTimer(anyString(), anyString())).thenReturn(timer);
    blobStoreTaskBackupMetrics = new BlobStoreBackupManagerMetrics(metricsRegistry);
    blobStoreBackupManager = new MockBlobStoreBackupManager(jobModel, containerModel, taskModel, mockExecutor, blobStoreTaskBackupMetrics, config, Files.createTempDirectory("logged-store-").toFile(), storageManagerUtil, blobStoreManager);
}

Example 65 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestStartpointManager method testBasics.

@Test
public void testBasics() {
    SystemStreamPartition ssp = new SystemStreamPartition("mockSystem", "mockStream", new Partition(2));
    TaskName taskName = new TaskName("MockTask");
    StartpointTimestamp startpoint1 = new StartpointTimestamp(111111111L);
    StartpointTimestamp startpoint2 = new StartpointTimestamp(222222222L);
    StartpointSpecific startpoint3 = new StartpointSpecific("1");
    StartpointSpecific startpoint4 = new StartpointSpecific("2");
    // Test createdTimestamp field is not null by default
    Assert.assertNotNull(startpoint1.getCreationTimestamp());
    Assert.assertNotNull(startpoint2.getCreationTimestamp());
    Assert.assertNotNull(startpoint3.getCreationTimestamp());
    Assert.assertNotNull(startpoint4.getCreationTimestamp());
    // Test reads on non-existent keys
    Assert.assertFalse(startpointManager.readStartpoint(ssp).isPresent());
    Assert.assertFalse(startpointManager.readStartpoint(ssp, taskName).isPresent());
    // Test writes
    Startpoint startpointFromStore;
    startpointManager.writeStartpoint(ssp, startpoint1);
    startpointManager.writeStartpoint(ssp, taskName, startpoint2);
    startpointFromStore = startpointManager.readStartpoint(ssp).get();
    Assert.assertEquals(StartpointTimestamp.class, startpointFromStore.getClass());
    Assert.assertEquals(startpoint1.getTimestampOffset(), ((StartpointTimestamp) startpointFromStore).getTimestampOffset());
    Assert.assertTrue(startpointFromStore.getCreationTimestamp() <= Instant.now().toEpochMilli());
    startpointFromStore = startpointManager.readStartpoint(ssp, taskName).get();
    Assert.assertEquals(StartpointTimestamp.class, startpointFromStore.getClass());
    Assert.assertEquals(startpoint2.getTimestampOffset(), ((StartpointTimestamp) startpointFromStore).getTimestampOffset());
    Assert.assertTrue(startpointFromStore.getCreationTimestamp() <= Instant.now().toEpochMilli());
    // Test overwrites
    startpointManager.writeStartpoint(ssp, startpoint3);
    startpointManager.writeStartpoint(ssp, taskName, startpoint4);
    startpointFromStore = startpointManager.readStartpoint(ssp).get();
    Assert.assertEquals(StartpointSpecific.class, startpointFromStore.getClass());
    Assert.assertEquals(startpoint3.getSpecificOffset(), ((StartpointSpecific) startpointFromStore).getSpecificOffset());
    Assert.assertTrue(startpointFromStore.getCreationTimestamp() <= Instant.now().toEpochMilli());
    startpointFromStore = startpointManager.readStartpoint(ssp, taskName).get();
    Assert.assertEquals(StartpointSpecific.class, startpointFromStore.getClass());
    Assert.assertEquals(startpoint4.getSpecificOffset(), ((StartpointSpecific) startpointFromStore).getSpecificOffset());
    Assert.assertTrue(startpointFromStore.getCreationTimestamp() <= Instant.now().toEpochMilli());
    // Test deletes on SSP keys does not affect SSP+TaskName keys
    startpointManager.deleteStartpoint(ssp);
    Assert.assertFalse(startpointManager.readStartpoint(ssp).isPresent());
    Assert.assertTrue(startpointManager.readStartpoint(ssp, taskName).isPresent());
    // Test deletes on SSP+TaskName keys does not affect SSP keys
    startpointManager.writeStartpoint(ssp, startpoint3);
    startpointManager.deleteStartpoint(ssp, taskName);
    Assert.assertFalse(startpointManager.readStartpoint(ssp, taskName).isPresent());
    Assert.assertTrue(startpointManager.readStartpoint(ssp).isPresent());
}