Examples with SegmentLoaderConfig org.apache.druid.segment.loading.SegmentLoaderConfig used on opensource projects

Example 1 with SegmentLoaderConfig

use of org.apache.druid.segment.loading.SegmentLoaderConfig in project druid by druid-io.

the class DruidServerConfigTest method testCombinedSize.

@Test
public void testCombinedSize() {
    final List<StorageLocationConfig> locations = new ArrayList<>();
    final StorageLocationConfig locationConfig1 = new StorageLocationConfig(testSegmentCacheDir1, 10000000000L, null);
    final StorageLocationConfig locationConfig2 = new StorageLocationConfig(testSegmentCacheDir2, 20000000000L, null);
    locations.add(locationConfig1);
    locations.add(locationConfig2);
    DruidServerConfig druidServerConfig = new DruidServerConfig(new SegmentLoaderConfig().withLocations(locations));
    Assert.assertEquals(30000000000L, druidServerConfig.getMaxSize());
}

Example 2 with SegmentLoaderConfig

use of org.apache.druid.segment.loading.SegmentLoaderConfig in project druid by druid-io.

the class SegmentLoadDropHandlerCacheTest method setup.

@Before
public void setup() throws IOException {
    storageLoc = new TestStorageLocation(temporaryFolder);
    SegmentLoaderConfig config = new SegmentLoaderConfig().withLocations(Collections.singletonList(storageLoc.toStorageLocationConfig(MAX_SIZE, null))).withInfoDir(storageLoc.getInfoDir());
    objectMapper = TestHelper.makeJsonMapper();
    objectMapper.registerSubtypes(TestLoadSpec.class);
    objectMapper.registerSubtypes(TestSegmentizerFactory.class);
    SegmentCacheManager cacheManager = new SegmentLocalCacheManager(config, objectMapper);
    SegmentManager segmentManager = new SegmentManager(new SegmentLocalCacheLoader(cacheManager, TestIndex.INDEX_IO, objectMapper));
    segmentAnnouncer = Mockito.mock(DataSegmentAnnouncer.class);
    loadDropHandler = new SegmentLoadDropHandler(objectMapper, config, segmentAnnouncer, Mockito.mock(DataSegmentServerAnnouncer.class), segmentManager, cacheManager, new ServerTypeConfig(ServerType.HISTORICAL));
    EmittingLogger.registerEmitter(new NoopServiceEmitter());
}

Example 3 with SegmentLoaderConfig

use of org.apache.druid.segment.loading.SegmentLoaderConfig in project druid by druid-io.

the class SegmentManagerBroadcastJoinIndexedTableTest method setup.

@Before
public void setup() throws IOException {
    segmentPuller = new LocalDataSegmentPuller();
    objectMapper = new DefaultObjectMapper().registerModule(new SegmentizerModule()).registerModule(new SimpleModule().registerSubtypes(new NamedType(LocalLoadSpec.class, "local")));
    indexIO = new IndexIO(objectMapper, () -> 0);
    objectMapper.setInjectableValues(new InjectableValues.Std().addValue(LocalDataSegmentPuller.class, segmentPuller).addValue(ExprMacroTable.class.getName(), TestExprMacroTable.INSTANCE).addValue(ObjectMapper.class.getName(), objectMapper).addValue(IndexIO.class, indexIO));
    segmentCacheDir = temporaryFolder.newFolder();
    segmentDeepStorageDir = temporaryFolder.newFolder();
    segmentCacheManager = new SegmentLocalCacheManager(new SegmentLoaderConfig() {

        @Override
        public List<StorageLocationConfig> getLocations() {
            return Collections.singletonList(new StorageLocationConfig(segmentCacheDir, null, null));
        }
    }, objectMapper);
    segmentManager = new SegmentManager(new SegmentLocalCacheLoader(segmentCacheManager, indexIO, objectMapper));
    joinableFactory = new BroadcastTableJoinableFactory(segmentManager);
    EmittingLogger.registerEmitter(new NoopServiceEmitter());
}

Example 4 with SegmentLoaderConfig

use of org.apache.druid.segment.loading.SegmentLoaderConfig in project druid by druid-io.

the class SegmentManagerThreadSafetyTest method setup.

@Before
public void setup() throws IOException {
    segmentPuller = new TestSegmentPuller();
    objectMapper = new DefaultObjectMapper().registerModule(new SimpleModule().registerSubtypes(new NamedType(LocalLoadSpec.class, "local"), new NamedType(TestSegmentizerFactory.class, "test"))).setInjectableValues(new Std().addValue(LocalDataSegmentPuller.class, segmentPuller));
    indexIO = new IndexIO(objectMapper, () -> 0);
    segmentCacheDir = temporaryFolder.newFolder();
    segmentDeepStorageDir = temporaryFolder.newFolder();
    segmentCacheManager = new SegmentLocalCacheManager(new SegmentLoaderConfig() {

        @Override
        public List<StorageLocationConfig> getLocations() {
            return Collections.singletonList(new StorageLocationConfig(segmentCacheDir, null, null));
        }
    }, objectMapper);
    segmentManager = new SegmentManager(new SegmentLocalCacheLoader(segmentCacheManager, indexIO, objectMapper));
    exec = Execs.multiThreaded(NUM_THREAD, "SegmentManagerThreadSafetyTest-%d");
    EmittingLogger.registerEmitter(new NoopServiceEmitter());
}

Example 5 with SegmentLoaderConfig

use of org.apache.druid.segment.loading.SegmentLoaderConfig in project druid by druid-io.

the class ZkCoordinatorTest method testLoadDrop.

@Test(timeout = 60_000L)
public void testLoadDrop() throws Exception {
    EmittingLogger.registerEmitter(new NoopServiceEmitter());
    DataSegment segment = new DataSegment("test", Intervals.of("P1d/2011-04-02"), "v0", ImmutableMap.of("version", "v0", "interval", Intervals.of("P1d/2011-04-02"), "cacheDir", "/no"), Arrays.asList("dim1", "dim2", "dim3"), Arrays.asList("metric1", "metric2"), NoneShardSpec.instance(), IndexIO.CURRENT_VERSION_ID, 123L);
    CountDownLatch loadLatch = new CountDownLatch(1);
    CountDownLatch dropLatch = new CountDownLatch(1);
    SegmentLoadDropHandler segmentLoadDropHandler = new SegmentLoadDropHandler(ServerTestHelper.MAPPER, new SegmentLoaderConfig() {

        @Override
        public File getInfoDir() {
            return infoDir;
        }

        @Override
        public int getNumLoadingThreads() {
            return 5;
        }

        @Override
        public int getAnnounceIntervalMillis() {
            return 50;
        }

        @Override
        public List<StorageLocationConfig> getLocations() {
            return locations;
        }

        @Override
        public int getDropSegmentDelayMillis() {
            return 0;
        }
    }, EasyMock.createNiceMock(DataSegmentAnnouncer.class), EasyMock.createNiceMock(DataSegmentServerAnnouncer.class), EasyMock.createNiceMock(SegmentManager.class), EasyMock.createNiceMock(SegmentCacheManager.class), EasyMock.createNiceMock(ScheduledExecutorService.class), new ServerTypeConfig(ServerType.HISTORICAL)) {

        @Override
        public void addSegment(DataSegment s, DataSegmentChangeCallback callback) {
            if (segment.getId().equals(s.getId())) {
                loadLatch.countDown();
                callback.execute();
            }
        }

        @Override
        public void removeSegment(DataSegment s, DataSegmentChangeCallback callback) {
            if (segment.getId().equals(s.getId())) {
                dropLatch.countDown();
                callback.execute();
            }
        }
    };
    zkCoordinator = new ZkCoordinator(segmentLoadDropHandler, jsonMapper, zkPaths, me, curator, new SegmentLoaderConfig());
    zkCoordinator.start();
    String segmentZkPath = ZKPaths.makePath(zkPaths.getLoadQueuePath(), me.getName(), segment.getId().toString());
    curator.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(segmentZkPath, jsonMapper.writeValueAsBytes(new SegmentChangeRequestLoad(segment)));
    loadLatch.await();
    while (curator.checkExists().forPath(segmentZkPath) != null) {
        Thread.sleep(100);
    }
    curator.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(segmentZkPath, jsonMapper.writeValueAsBytes(new SegmentChangeRequestDrop(segment)));
    dropLatch.await();
    while (curator.checkExists().forPath(segmentZkPath) != null) {
        Thread.sleep(100);
    }
    zkCoordinator.stop();
}