Examples with IndexerMetadataStorageCoordinator io.druid.indexing.overlord.IndexerMetadataStorageCoordinator used on opensource projects

Example 1 with IndexerMetadataStorageCoordinator

use of io.druid.indexing.overlord.IndexerMetadataStorageCoordinator in project druid by druid-io.

the class KafkaSupervisorTest method setUp.

@Before
public void setUp() throws Exception {
    taskStorage = createMock(TaskStorage.class);
    taskMaster = createMock(TaskMaster.class);
    taskRunner = createMock(TaskRunner.class);
    indexerMetadataStorageCoordinator = createMock(IndexerMetadataStorageCoordinator.class);
    taskClient = createMock(KafkaIndexTaskClient.class);
    taskQueue = createMock(TaskQueue.class);
    zkServer = new TestingCluster(1);
    zkServer.start();
    kafkaServer = new TestBroker(zkServer.getConnectString(), tempFolder.newFolder(), 1, ImmutableMap.of("num.partitions", String.valueOf(NUM_PARTITIONS)));
    kafkaServer.start();
    kafkaHost = String.format("localhost:%d", kafkaServer.getPort());
    dataSchema = getDataSchema(DATASOURCE);
    tuningConfig = new KafkaSupervisorTuningConfig(1000, 50000, new Period("P1Y"), new File("/test"), null, null, true, false, null, null, numThreads, TEST_CHAT_THREADS, TEST_CHAT_RETRIES, TEST_HTTP_TIMEOUT, TEST_SHUTDOWN_TIMEOUT);
}

Example 2 with IndexerMetadataStorageCoordinator

use of io.druid.indexing.overlord.IndexerMetadataStorageCoordinator in project druid by druid-io.

the class SegmentAllocateAction method perform.

@Override
public SegmentIdentifier perform(final Task task, final TaskActionToolbox toolbox) throws IOException {
    int attempt = 0;
    while (true) {
        attempt++;
        if (!task.getDataSource().equals(dataSource)) {
            throw new IAE("Task dataSource must match action dataSource, [%s] != [%s].", task.getDataSource(), dataSource);
        }
        final IndexerMetadataStorageCoordinator msc = toolbox.getIndexerMetadataStorageCoordinator();
        // 1) if something overlaps our timestamp, use that
        // 2) otherwise try preferredSegmentGranularity & going progressively smaller
        final List<Interval> tryIntervals = Lists.newArrayList();
        final Interval rowInterval = queryGranularity.bucket(timestamp);
        final Set<DataSegment> usedSegmentsForRow = ImmutableSet.copyOf(msc.getUsedSegmentsForInterval(dataSource, rowInterval));
        if (usedSegmentsForRow.isEmpty()) {
            // segment granularity. Try that first, and then progressively smaller ones if it fails.
            for (Granularity gran : Granularity.granularitiesFinerThan(preferredSegmentGranularity)) {
                tryIntervals.add(gran.bucket(timestamp));
            }
        } else {
            // Existing segment(s) exist for this row; use the interval of the first one.
            tryIntervals.add(usedSegmentsForRow.iterator().next().getInterval());
        }
        for (final Interval tryInterval : tryIntervals) {
            if (tryInterval.contains(rowInterval)) {
                log.debug("Trying to allocate pending segment for rowInterval[%s], segmentInterval[%s].", rowInterval, tryInterval);
                final TaskLock tryLock = toolbox.getTaskLockbox().tryLock(task, tryInterval).orNull();
                if (tryLock != null) {
                    final SegmentIdentifier identifier = msc.allocatePendingSegment(dataSource, sequenceName, previousSegmentId, tryInterval, tryLock.getVersion());
                    if (identifier != null) {
                        return identifier;
                    } else {
                        log.debug("Could not allocate pending segment for rowInterval[%s], segmentInterval[%s].", rowInterval, tryInterval);
                    }
                } else {
                    log.debug("Could not acquire lock for rowInterval[%s], segmentInterval[%s].", rowInterval, tryInterval);
                }
            }
        }
        if (!ImmutableSet.copyOf(msc.getUsedSegmentsForInterval(dataSource, rowInterval)).equals(usedSegmentsForRow)) {
            if (attempt < MAX_ATTEMPTS) {
                final long shortRandomSleep = 50 + (long) (Math.random() * 450);
                log.debug("Used segment set changed for rowInterval[%s]. Retrying segment allocation in %,dms (attempt = %,d).", rowInterval, shortRandomSleep, attempt);
                try {
                    Thread.sleep(shortRandomSleep);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                    throw Throwables.propagate(e);
                }
            } else {
                log.error("Used segment set changed for rowInterval[%s]. Not trying again (attempt = %,d).", rowInterval, attempt);
                return null;
            }
        } else {
            return null;
        }
    }
}

Example 3 with IndexerMetadataStorageCoordinator

use of io.druid.indexing.overlord.IndexerMetadataStorageCoordinator in project druid by druid-io.

the class InsertSegment method run.

@Override
public void run() {
    final Injector injector = makeInjector();
    mapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
    indexerMetadataStorageCoordinator = injector.getInstance(IndexerMetadataStorageCoordinator.class);
    final DataSegmentFinder dataSegmentFinder = injector.getInstance(DataSegmentFinder.class);
    log.info("Start seraching segments under [%s]", workingDirPath);
    Set<DataSegment> segments = null;
    try {
        segments = dataSegmentFinder.findSegments(workingDirPath, Boolean.valueOf(updateDescriptor));
    } catch (SegmentLoadingException e) {
        Throwables.propagate(e);
    }
    log.info("Done searching segments under [%s], [%d] segments were found", workingDirPath, segments.size());
    try {
        insertSegments(segments);
    } catch (IOException e) {
        Throwables.propagate(e);
    }
    log.info("Done processing [%d] segments", segments.size());
}