Examples with SegmentId org.apache.druid.timeline.SegmentId used on opensource projects

Example 1 with SegmentId

use of org.apache.druid.timeline.SegmentId in project druid by druid-io.

the class DataSourcesResource method computeSegmentLoadStatistics.

private SegmentsLoadStatistics computeSegmentLoadStatistics(Iterable<DataSegment> segments) {
    Map<SegmentId, SegmentLoadInfo> segmentLoadInfos = serverInventoryView.getSegmentLoadInfos();
    int numPublishedSegments = 0;
    int numUnavailableSegments = 0;
    int numLoadedSegments = 0;
    for (DataSegment segment : segments) {
        numPublishedSegments++;
        if (!segmentLoadInfos.containsKey(segment.getId())) {
            numUnavailableSegments++;
        } else {
            numLoadedSegments++;
        }
    }
    return new SegmentsLoadStatistics(numPublishedSegments, numUnavailableSegments, numLoadedSegments);
}

Example 2 with SegmentId

use of org.apache.druid.timeline.SegmentId in project druid by druid-io.

the class DataSourcesResource method markSegmentAsUnused.

@DELETE
@Path("/{dataSourceName}/segments/{segmentId}")
@ResourceFilters(DatasourceResourceFilter.class)
public Response markSegmentAsUnused(@PathParam("dataSourceName") String dataSourceName, @PathParam("segmentId") String segmentIdString) {
    final SegmentId segmentId = SegmentId.tryParse(dataSourceName, segmentIdString);
    final boolean segmentStateChanged = segmentId != null && segmentsMetadataManager.markSegmentAsUnused(segmentId);
    return Response.ok(ImmutableMap.of("segmentStateChanged", segmentStateChanged)).build();
}

Example 3 with SegmentId

use of org.apache.druid.timeline.SegmentId in project druid by druid-io.

the class DataSourcesResource method getDataSource.

@Nullable
private ImmutableDruidDataSource getDataSource(final String dataSourceName) {
    List<DruidDataSource> dataSources = serverInventoryView.getInventory().stream().map(server -> server.getDataSource(dataSourceName)).filter(Objects::nonNull).collect(Collectors.toList());
    if (dataSources.isEmpty()) {
        return null;
    }
    // Note: this logic doesn't guarantee that the result is a snapshot that ever existed in the cluster because all
    // DruidDataSource objects (belonging to different servers) are independently, concurrently mutable objects.
    // But this is OK because a "snapshot" hardly even makes sense in a distributed system anyway.
    final SortedMap<SegmentId, DataSegment> segmentMap = new TreeMap<>();
    for (DruidDataSource dataSource : dataSources) {
        Iterable<DataSegment> segments = dataSource.getSegments();
        for (DataSegment segment : segments) {
            segmentMap.put(segment.getId(), segment);
        }
    }
    return new ImmutableDruidDataSource(dataSourceName, Collections.emptyMap(), segmentMap);
}

Example 4 with SegmentId

use of org.apache.druid.timeline.SegmentId in project druid by druid-io.

the class LoadRule method assignReplicasForTier.

/**
 * @param predicate {@link Predicate} used to pre-filter {@link ServerHolder}s retrieved from {@link DruidCluster}.
 */
private int assignReplicasForTier(final String tier, final int targetReplicantsInTier, final int currentReplicantsInTier, final DruidCoordinatorRuntimeParams params, final Predicate<ServerHolder> predicate, final DataSegment segment) {
    final int numToAssign = targetReplicantsInTier - currentReplicantsInTier;
    // if nothing to assign
    if (numToAssign <= 0) {
        return 0;
    }
    String noAvailability = StringUtils.format("No available [%s] servers or node capacity to assign segment[%s]! Expected Replicants[%d]", tier, segment.getId(), targetReplicantsInTier);
    final List<ServerHolder> holders = getFilteredHolders(tier, params.getDruidCluster(), predicate);
    // if no holders available for assignment
    if (holders.isEmpty()) {
        log.warn(noAvailability);
        return 0;
    }
    final ReplicationThrottler throttler = params.getReplicationManager();
    for (int numAssigned = 0; numAssigned < numToAssign; numAssigned++) {
        if (!throttler.canCreateReplicant(tier)) {
            log.info("Throttling replication for segment [%s] in tier [%s]", segment.getId(), tier);
            return numAssigned;
        }
        // Retrieves from cache if available
        ServerHolder holder = strategyCache.remove(tier);
        // Does strategy call if not in cache
        if (holder == null) {
            holder = params.getBalancerStrategy().findNewSegmentHomeReplicator(segment, holders);
        }
        if (holder == null) {
            log.warn(noAvailability);
            return numAssigned;
        }
        holders.remove(holder);
        final SegmentId segmentId = segment.getId();
        final String holderHost = holder.getServer().getHost();
        throttler.registerReplicantCreation(tier, segmentId, holderHost);
        log.info("Assigning 'replica' for segment [%s] to server [%s] in tier [%s]", segment.getId(), holder.getServer().getName(), holder.getServer().getTier());
        holder.getPeon().loadSegment(segment, () -> throttler.unregisterReplicantCreation(tier, segmentId));
    }
    return numToAssign;
}

Example 5 with SegmentId

use of org.apache.druid.timeline.SegmentId in project druid by druid-io.

the class DataSourcesResourceTest method testGetDatasourceLoadstatusDefault.

@Test
public void testGetDatasourceLoadstatusDefault() {
    DataSegment datasource1Segment1 = new DataSegment("datasource1", Intervals.of("2010-01-01/P1D"), "", null, null, null, null, 0x9, 10);
    DataSegment datasource1Segment2 = new DataSegment("datasource1", Intervals.of("2010-01-22/P1D"), "", null, null, null, null, 0x9, 20);
    DataSegment datasource2Segment1 = new DataSegment("datasource2", Intervals.of("2010-01-01/P1D"), "", null, null, null, null, 0x9, 30);
    List<DataSegment> segments = ImmutableList.of(datasource1Segment1, datasource1Segment2);
    Map<SegmentId, SegmentLoadInfo> completedLoadInfoMap = ImmutableMap.of(datasource1Segment1.getId(), new SegmentLoadInfo(datasource1Segment1), datasource1Segment2.getId(), new SegmentLoadInfo(datasource1Segment2), datasource2Segment1.getId(), new SegmentLoadInfo(datasource2Segment1));
    Map<SegmentId, SegmentLoadInfo> halfLoadedInfoMap = ImmutableMap.of(datasource1Segment1.getId(), new SegmentLoadInfo(datasource1Segment1));
    // Test when datasource fully loaded
    EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())).andReturn(Optional.of(segments)).once();
    EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(completedLoadInfoMap).once();
    EasyMock.replay(segmentsMetadataManager, inventoryView);
    DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null);
    Response response = dataSourcesResource.getDatasourceLoadstatus("datasource1", true, null, null, null, null);
    Assert.assertEquals(200, response.getStatus());
    Assert.assertNotNull(response.getEntity());
    Assert.assertEquals(1, ((Map) response.getEntity()).size());
    Assert.assertTrue(((Map) response.getEntity()).containsKey("datasource1"));
    Assert.assertEquals(100.0, ((Map) response.getEntity()).get("datasource1"));
    EasyMock.verify(segmentsMetadataManager, inventoryView);
    EasyMock.reset(segmentsMetadataManager, inventoryView);
    // Test when datasource half loaded
    EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())).andReturn(Optional.of(segments)).once();
    EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(halfLoadedInfoMap).once();
    EasyMock.replay(segmentsMetadataManager, inventoryView);
    dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null);
    response = dataSourcesResource.getDatasourceLoadstatus("datasource1", true, null, null, null, null);
    Assert.assertEquals(200, response.getStatus());
    Assert.assertNotNull(response.getEntity());
    Assert.assertEquals(1, ((Map) response.getEntity()).size());
    Assert.assertTrue(((Map) response.getEntity()).containsKey("datasource1"));
    Assert.assertEquals(50.0, ((Map) response.getEntity()).get("datasource1"));
    EasyMock.verify(segmentsMetadataManager, inventoryView);
}