Examples with NoopSupervisorSpec org.apache.druid.indexing.overlord.supervisor.NoopSupervisorSpec used on opensource projects

Example 1 with NoopSupervisorSpec

use of org.apache.druid.indexing.overlord.supervisor.NoopSupervisorSpec in project druid by druid-io.

the class SQLMetadataSupervisorManagerTest method testRemoveTerminatedSupervisorsOlderThanWithSupervisorTerminatedAndOlderThanTimeShouldBeDeleted.

@Test
public void testRemoveTerminatedSupervisorsOlderThanWithSupervisorTerminatedAndOlderThanTimeShouldBeDeleted() {
    final String supervisor1 = "test-supervisor-1";
    final String datasource1 = "datasource-1";
    final Map<String, String> data1rev1 = ImmutableMap.of("key1-1", "value1-1-1", "key1-2", "value1-2-1");
    Assert.assertTrue(supervisorManager.getAll().isEmpty());
    supervisorManager.insert(supervisor1, new TestSupervisorSpec(supervisor1, data1rev1));
    supervisorManager.insert(supervisor1, new NoopSupervisorSpec(supervisor1, ImmutableList.of(datasource1)));
    // Test that supervisor was inserted
    Map<String, List<VersionedSupervisorSpec>> supervisorSpecs = supervisorManager.getAll();
    Assert.assertEquals(1, supervisorSpecs.size());
    Assert.assertEquals(2, supervisorSpecs.get(supervisor1).size());
    Map<String, SupervisorSpec> latestSpecs = supervisorManager.getLatest();
    Assert.assertEquals(1, latestSpecs.size());
    Assert.assertEquals(ImmutableList.of(datasource1), ((NoopSupervisorSpec) latestSpecs.get(supervisor1)).getDataSources());
    // Do delete. Supervisor should be deleted as it is terminated
    int deleteCount = supervisorManager.removeTerminatedSupervisorsOlderThan(System.currentTimeMillis());
    // Verify that supervisor was actually deleted
    Assert.assertEquals(2, deleteCount);
    supervisorSpecs = supervisorManager.getAll();
    Assert.assertEquals(0, supervisorSpecs.size());
    latestSpecs = supervisorManager.getLatest();
    Assert.assertEquals(0, latestSpecs.size());
}

Example 2 with NoopSupervisorSpec

use of org.apache.druid.indexing.overlord.supervisor.NoopSupervisorSpec in project druid by druid-io.

the class SQLMetadataSupervisorManagerTest method testGetLatestActiveOnly.

@Test
public void testGetLatestActiveOnly() {
    final String supervisor1 = "test-supervisor-1";
    final String datasource1 = "datasource-1";
    final String supervisor2 = "test-supervisor-2";
    final Map<String, String> data1rev1 = ImmutableMap.of("key1-1", "value1-1-1", "key1-2", "value1-2-1");
    Assert.assertTrue(supervisorManager.getAll().isEmpty());
    supervisorManager.insert(supervisor1, new TestSupervisorSpec(supervisor1, data1rev1));
    // supervisor1 is terminated
    supervisorManager.insert(supervisor1, new NoopSupervisorSpec(supervisor1, ImmutableList.of(datasource1)));
    // supervisor2 is still active
    supervisorManager.insert(supervisor2, new TestSupervisorSpec(supervisor2, data1rev1));
    // get latest active should only return supervisor2
    Map<String, SupervisorSpec> actual = supervisorManager.getLatestActiveOnly();
    Assert.assertEquals(1, actual.size());
    Assert.assertTrue(actual.containsKey(supervisor2));
}

Example 3 with NoopSupervisorSpec

use of org.apache.druid.indexing.overlord.supervisor.NoopSupervisorSpec in project druid by druid-io.

the class SQLMetadataSupervisorManagerTest method testRemoveTerminatedSupervisorsOlderThanWithSupervisorTerminatedButNotOlderThanTimeShouldNotBeDeleted.

@Test
public void testRemoveTerminatedSupervisorsOlderThanWithSupervisorTerminatedButNotOlderThanTimeShouldNotBeDeleted() {
    final String supervisor1 = "test-supervisor-1";
    final String datasource1 = "datasource-1";
    final Map<String, String> data1rev1 = ImmutableMap.of("key1-1", "value1-1-1", "key1-2", "value1-2-1");
    Assert.assertTrue(supervisorManager.getAll().isEmpty());
    supervisorManager.insert(supervisor1, new TestSupervisorSpec(supervisor1, data1rev1));
    supervisorManager.insert(supervisor1, new NoopSupervisorSpec(supervisor1, ImmutableList.of(datasource1)));
    // Test that supervisor was inserted
    Map<String, List<VersionedSupervisorSpec>> supervisorSpecs = supervisorManager.getAll();
    Assert.assertEquals(1, supervisorSpecs.size());
    Assert.assertEquals(2, supervisorSpecs.get(supervisor1).size());
    Map<String, SupervisorSpec> latestSpecs = supervisorManager.getLatest();
    Assert.assertEquals(1, latestSpecs.size());
    Assert.assertEquals(ImmutableList.of(datasource1), ((NoopSupervisorSpec) latestSpecs.get(supervisor1)).getDataSources());
    // Do delete. Supervisor should not be deleted. Supervisor is terminated but it was created just now so it's
    // created timestamp will be later than the timestamp 2012-01-01T00:00:00Z
    int deleteCount = supervisorManager.removeTerminatedSupervisorsOlderThan(DateTimes.of("2012-01-01T00:00:00Z").getMillis());
    // Verify that supervisor was not deleted
    Assert.assertEquals(0, deleteCount);
    supervisorSpecs = supervisorManager.getAll();
    Assert.assertEquals(1, supervisorSpecs.size());
    Assert.assertEquals(2, supervisorSpecs.get(supervisor1).size());
    latestSpecs = supervisorManager.getLatest();
    Assert.assertEquals(1, latestSpecs.size());
    Assert.assertEquals(ImmutableList.of(datasource1), ((NoopSupervisorSpec) latestSpecs.get(supervisor1)).getDataSources());
}

Example 4 with NoopSupervisorSpec

use of org.apache.druid.indexing.overlord.supervisor.NoopSupervisorSpec in project druid by druid-io.

the class NoopSupervisorSpecTest method testNoopSupervisorSpecWithAutoscaler.

@Test
public void testNoopSupervisorSpecWithAutoscaler() {
    Exception e = null;
    try {
        NoopSupervisorSpec noopSupervisorSpec = new NoopSupervisorSpec(null, Collections.singletonList("datasource1"));
        Supervisor supervisor = noopSupervisorSpec.createSupervisor();
        SupervisorTaskAutoScaler autoscaler = noopSupervisorSpec.createAutoscaler(supervisor);
        Assert.assertNull(autoscaler);
        Callable<Integer> noop = new Callable<Integer>() {

            @Override
            public Integer call() {
                return -1;
            }
        };
        int count = supervisor.getActiveTaskGroupsCount();
        Assert.assertEquals(count, -1);
        LagStats lagStats = supervisor.computeLagStats();
        long totalLag = lagStats.getTotalLag();
        long avgLag = lagStats.getAvgLag();
        long maxLag = lagStats.getMaxLag();
        Assert.assertEquals(totalLag, 0);
        Assert.assertEquals(avgLag, 0);
        Assert.assertEquals(maxLag, 0);
    } catch (Exception ex) {
        e = ex;
    }
    Assert.assertNull(e);
}

Example 5 with NoopSupervisorSpec

use of org.apache.druid.indexing.overlord.supervisor.NoopSupervisorSpec in project druid by druid-io.

the class SQLMetadataSupervisorManagerTest method testGetLatestTerminatedOnly.

@Test
public void testGetLatestTerminatedOnly() {
    final String supervisor1 = "test-supervisor-1";
    final String datasource1 = "datasource-1";
    final String supervisor2 = "test-supervisor-2";
    final Map<String, String> data1rev1 = ImmutableMap.of("key1-1", "value1-1-1", "key1-2", "value1-2-1");
    Assert.assertTrue(supervisorManager.getAll().isEmpty());
    supervisorManager.insert(supervisor1, new TestSupervisorSpec(supervisor1, data1rev1));
    // supervisor1 is terminated
    supervisorManager.insert(supervisor1, new NoopSupervisorSpec(supervisor1, ImmutableList.of(datasource1)));
    // supervisor2 is still active
    supervisorManager.insert(supervisor2, new TestSupervisorSpec(supervisor2, data1rev1));
    // get latest terminated should only return supervisor1
    Map<String, SupervisorSpec> actual = supervisorManager.getLatestTerminatedOnly();
    Assert.assertEquals(1, actual.size());
    Assert.assertTrue(actual.containsKey(supervisor1));
}