Examples with ZkPathsConfig io.druid.server.initialization.ZkPathsConfig used on opensource projects

Example 1 with ZkPathsConfig

use of io.druid.server.initialization.ZkPathsConfig in project druid by druid-io.

the class BatchServerInventoryViewTest method setUp.

@Before
public void setUp() throws Exception {
    testingCluster = new TestingCluster(1);
    testingCluster.start();
    cf = CuratorFrameworkFactory.builder().connectString(testingCluster.getConnectString()).retryPolicy(new ExponentialBackoffRetry(1, 10)).compressionProvider(new PotentiallyGzippedCompressionProvider(true)).build();
    cf.start();
    cf.blockUntilConnected();
    cf.create().creatingParentsIfNeeded().forPath(testBasePath);
    jsonMapper = new DefaultObjectMapper();
    announcer = new Announcer(cf, MoreExecutors.sameThreadExecutor());
    announcer.start();
    segmentAnnouncer = new BatchDataSegmentAnnouncer(new DruidServerMetadata("id", "host", Long.MAX_VALUE, "type", "tier", 0), new BatchDataSegmentAnnouncerConfig() {

        @Override
        public int getSegmentsPerNode() {
            return 50;
        }
    }, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return testBasePath;
        }
    }, announcer, jsonMapper);
    segmentAnnouncer.start();
    testSegments = Sets.newConcurrentHashSet();
    for (int i = 0; i < INITIAL_SEGMENTS; i++) {
        testSegments.add(makeSegment(i));
    }
    batchServerInventoryView = new BatchServerInventoryView(new ZkPathsConfig() {

        @Override
        public String getBase() {
            return testBasePath;
        }
    }, cf, jsonMapper, Predicates.<Pair<DruidServerMetadata, DataSegment>>alwaysTrue());
    batchServerInventoryView.start();
    inventoryUpdateCounter.set(0);
    filteredBatchServerInventoryView = new BatchServerInventoryView(new ZkPathsConfig() {

        @Override
        public String getBase() {
            return testBasePath;
        }
    }, cf, jsonMapper, new Predicate<Pair<DruidServerMetadata, DataSegment>>() {

        @Override
        public boolean apply(@Nullable Pair<DruidServerMetadata, DataSegment> input) {
            return input.rhs.getInterval().getStart().isBefore(SEGMENT_INTERVAL_START.plusDays(INITIAL_SEGMENTS));
        }
    }) {

        @Override
        protected DruidServer addInnerInventory(DruidServer container, String inventoryKey, Set<DataSegment> inventory) {
            DruidServer server = super.addInnerInventory(container, inventoryKey, inventory);
            inventoryUpdateCounter.incrementAndGet();
            return server;
        }
    };
    filteredBatchServerInventoryView.start();
}

Example 2 with ZkPathsConfig

use of io.druid.server.initialization.ZkPathsConfig in project druid by druid-io.

the class ZkCoordinatorTest method testInjector.

@Test
public void testInjector() throws Exception {
    Injector injector = Guice.createInjector(new Module() {

        @Override
        public void configure(Binder binder) {
            binder.bind(ObjectMapper.class).toInstance(jsonMapper);
            binder.bind(SegmentLoaderConfig.class).toInstance(new SegmentLoaderConfig() {

                @Override
                public File getInfoDir() {
                    return infoDir;
                }

                @Override
                public int getNumLoadingThreads() {
                    return 5;
                }

                @Override
                public int getAnnounceIntervalMillis() {
                    return 50;
                }
            });
            binder.bind(ZkPathsConfig.class).toInstance(new ZkPathsConfig() {

                @Override
                public String getBase() {
                    return "/druid";
                }
            });
            binder.bind(DruidServerMetadata.class).toInstance(new DruidServerMetadata("dummyServer", "dummyHost", 0, "dummyType", "normal", 0));
            binder.bind(DataSegmentAnnouncer.class).toInstance(announcer);
            binder.bind(CuratorFramework.class).toInstance(curator);
            binder.bind(ServerManager.class).toInstance(serverManager);
            binder.bind(ScheduledExecutorFactory.class).toInstance(ScheduledExecutors.createFactory(new Lifecycle()));
        }
    });
    ZkCoordinator zkCoordinator = injector.getInstance(ZkCoordinator.class);
    List<DataSegment> segments = Lists.newLinkedList();
    for (int i = 0; i < COUNT; ++i) {
        segments.add(makeSegment("test" + i, "1", new Interval("P1d/2011-04-01")));
        segments.add(makeSegment("test" + i, "1", new Interval("P1d/2011-04-02")));
        segments.add(makeSegment("test" + i, "2", new Interval("P1d/2011-04-02")));
        segments.add(makeSegment("test_two" + i, "1", new Interval("P1d/2011-04-01")));
        segments.add(makeSegment("test_two" + i, "1", new Interval("P1d/2011-04-02")));
    }
    Collections.sort(segments);
    for (DataSegment segment : segments) {
        writeSegmentToCache(segment);
    }
    checkCache(segments);
    Assert.assertTrue(serverManager.getDataSourceCounts().isEmpty());
    zkCoordinator.start();
    Assert.assertTrue(!serverManager.getDataSourceCounts().isEmpty());
    for (int i = 0; i < COUNT; ++i) {
        Assert.assertEquals(3L, serverManager.getDataSourceCounts().get("test" + i).longValue());
        Assert.assertEquals(2L, serverManager.getDataSourceCounts().get("test_two" + i).longValue());
    }
    Assert.assertEquals(5 * COUNT, announceCount.get());
    zkCoordinator.stop();
    for (DataSegment segment : segments) {
        deleteSegmentFromCache(segment);
    }
    Assert.assertEquals(0, infoDir.listFiles().length);
    Assert.assertTrue(infoDir.delete());
}

Example 3 with ZkPathsConfig

use of io.druid.server.initialization.ZkPathsConfig in project druid by druid-io.

the class RemoteTaskRunnerFactoryTest method testExecNotSharedBetweenRunners.

@Test
public void testExecNotSharedBetweenRunners() {
    final AtomicInteger executorCount = new AtomicInteger(0);
    RemoteTaskRunnerConfig config = new RemoteTaskRunnerConfig();
    IndexerZkConfig indexerZkConfig = new IndexerZkConfig(new ZkPathsConfig() {

        @Override
        public String getBase() {
            return basePath;
        }
    }, null, null, null, null, null);
    HttpClient httpClient = EasyMock.createMock(HttpClient.class);
    Supplier<WorkerBehaviorConfig> workerBehaviorConfig = EasyMock.createMock(Supplier.class);
    ScheduledExecutorFactory executorFactory = new ScheduledExecutorFactory() {

        @Override
        public ScheduledExecutorService create(int i, String s) {
            executorCount.incrementAndGet();
            return ScheduledExecutors.fixed(i, s);
        }
    };
    SimpleWorkerResourceManagementConfig resourceManagementConfig = new SimpleWorkerResourceManagementConfig();
    ResourceManagementSchedulerConfig resourceManagementSchedulerConfig = new ResourceManagementSchedulerConfig() {

        @Override
        public boolean isDoAutoscale() {
            return true;
        }
    };
    RemoteTaskRunnerFactory factory = new RemoteTaskRunnerFactory(cf, config, indexerZkConfig, jsonMapper, httpClient, workerBehaviorConfig, executorFactory, resourceManagementSchedulerConfig, new SimpleWorkerResourceManagementStrategy(resourceManagementConfig, workerBehaviorConfig, resourceManagementSchedulerConfig, executorFactory));
    Assert.assertEquals(1, executorCount.get());
    RemoteTaskRunner remoteTaskRunner1 = factory.build();
    Assert.assertEquals(2, executorCount.get());
    RemoteTaskRunner remoteTaskRunner2 = factory.build();
    Assert.assertEquals(3, executorCount.get());
}

Example 4 with ZkPathsConfig

use of io.druid.server.initialization.ZkPathsConfig in project druid by druid-io.

the class WorkerTaskMonitorTest method setUp.

@Before
public void setUp() throws Exception {
    testingCluster = new TestingCluster(1);
    testingCluster.start();
    cf = CuratorFrameworkFactory.builder().connectString(testingCluster.getConnectString()).retryPolicy(new ExponentialBackoffRetry(1, 10)).compressionProvider(new PotentiallyGzippedCompressionProvider(false)).build();
    cf.start();
    cf.blockUntilConnected();
    cf.create().creatingParentsIfNeeded().forPath(basePath);
    worker = new Worker("worker", "localhost", 3, "0");
    workerCuratorCoordinator = new WorkerCuratorCoordinator(jsonMapper, new IndexerZkConfig(new ZkPathsConfig() {

        @Override
        public String getBase() {
            return basePath;
        }
    }, null, null, null, null, null), new TestRemoteTaskRunnerConfig(new Period("PT1S")), cf, worker);
    workerCuratorCoordinator.start();
    // Start a task monitor
    workerTaskMonitor = createTaskMonitor();
    TestTasks.registerSubtypes(jsonMapper);
    jsonMapper.registerSubtypes(new NamedType(TestRealtimeTask.class, "test_realtime"));
    workerTaskMonitor.start();
    task = TestTasks.immediateSuccess("test");
}

Example 5 with ZkPathsConfig

use of io.druid.server.initialization.ZkPathsConfig in project druid by druid-io.

the class WorkerResourceTest method setUp.

@Before
public void setUp() throws Exception {
    testingCluster = new TestingCluster(1);
    testingCluster.start();
    cf = CuratorFrameworkFactory.builder().connectString(testingCluster.getConnectString()).retryPolicy(new ExponentialBackoffRetry(1, 10)).compressionProvider(new PotentiallyGzippedCompressionProvider(false)).build();
    cf.start();
    cf.blockUntilConnected();
    cf.create().creatingParentsIfNeeded().forPath(basePath);
    worker = new Worker("host", "ip", 3, "v1");
    curatorCoordinator = new WorkerCuratorCoordinator(jsonMapper, new IndexerZkConfig(new ZkPathsConfig() {

        @Override
        public String getBase() {
            return basePath;
        }
    }, null, null, null, null, null), new RemoteTaskRunnerConfig(), cf, worker);
    curatorCoordinator.start();
    workerResource = new WorkerResource(worker, curatorCoordinator, null);
}