Examples with Interval org.joda.time.Interval used on opensource projects

Example 11 with Interval

use of org.joda.time.Interval in project druid by druid-io.

the class QuotableWhiteSpaceSplitter method stop.

@LifecycleStop
public void stop() {
    stopping = true;
    exec.shutdown();
    synchronized (tasks) {
        for (ForkingTaskRunnerWorkItem taskWorkItem : tasks.values()) {
            if (taskWorkItem.processHolder != null) {
                log.info("Closing output stream to task[%s].", taskWorkItem.getTask().getId());
                try {
                    taskWorkItem.processHolder.process.getOutputStream().close();
                } catch (Exception e) {
                    log.warn(e, "Failed to close stdout to task[%s]. Destroying task.", taskWorkItem.getTask().getId());
                    taskWorkItem.processHolder.process.destroy();
                }
            }
        }
    }
    final DateTime start = new DateTime();
    final long timeout = new Interval(start, taskConfig.getGracefulShutdownTimeout()).toDurationMillis();
    // Things should be terminating now. Wait for it to happen so logs can be uploaded and all that good stuff.
    log.info("Waiting up to %,dms for shutdown.", timeout);
    if (timeout > 0) {
        try {
            final boolean terminated = exec.awaitTermination(timeout, TimeUnit.MILLISECONDS);
            final long elapsed = System.currentTimeMillis() - start.getMillis();
            if (terminated) {
                log.info("Finished stopping in %,dms.", elapsed);
            } else {
                final Set<String> stillRunning;
                synchronized (tasks) {
                    stillRunning = ImmutableSet.copyOf(tasks.keySet());
                }
                log.makeAlert("Failed to stop forked tasks").addData("stillRunning", stillRunning).addData("elapsed", elapsed).emit();
                log.warn("Executor failed to stop after %,dms, not waiting for it! Tasks still running: [%s]", elapsed, Joiner.on("; ").join(stillRunning));
            }
        } catch (InterruptedException e) {
            log.warn(e, "Interrupted while waiting for executor to finish.");
            Thread.currentThread().interrupt();
        }
    } else {
        log.warn("Ran out of time, not waiting for executor to finish!");
    }
}

Example 12 with Interval

use of org.joda.time.Interval in project druid by druid-io.

the class SameIntervalMergeTaskTest method testRun.

@Test
public void testRun() throws Exception {
    final List<AggregatorFactory> aggregators = ImmutableList.<AggregatorFactory>of(new CountAggregatorFactory("cnt"));
    final SameIntervalMergeTask task = new SameIntervalMergeTask(null, "foo", new Interval("2010-01-01/P1D"), aggregators, true, indexSpec, true, null);
    String newVersion = "newVersion";
    final List<DataSegment> segments = runTask(task, newVersion);
    // the lock is acquired
    Assert.assertEquals(0, isRedayCountDown.getCount());
    // the merged segment is published
    Assert.assertEquals(0, publishCountDown.getCount());
    // the merged segment is the only element
    Assert.assertEquals(1, segments.size());
    DataSegment mergeSegment = segments.get(0);
    Assert.assertEquals("foo", mergeSegment.getDataSource());
    Assert.assertEquals(newVersion, mergeSegment.getVersion());
    // the merged segment's interval is within the requested interval
    Assert.assertTrue(new Interval("2010-01-01/P1D").contains(mergeSegment.getInterval()));
    // the merged segment should be NoneShardSpec
    Assert.assertTrue(mergeSegment.getShardSpec() instanceof NoneShardSpec);
}

Example 13 with Interval

use of org.joda.time.Interval in project druid by druid-io.

the class SameIntervalMergeTaskTest method runTask.

private List<DataSegment> runTask(final SameIntervalMergeTask mergeTask, final String version) throws Exception {
    boolean isReady = mergeTask.isReady(new TaskActionClient() {

        @Override
        public <RetType> RetType submit(TaskAction<RetType> taskAction) throws IOException {
            if (taskAction instanceof LockTryAcquireAction) {
                // the lock of this interval is required
                Assert.assertEquals(mergeTask.getInterval(), ((LockTryAcquireAction) taskAction).getInterval());
                isRedayCountDown.countDown();
                taskLock = new TaskLock(mergeTask.getGroupId(), mergeTask.getDataSource(), mergeTask.getInterval(), version);
                return (RetType) taskLock;
            }
            return null;
        }
    });
    // ensure LockTryAcquireAction is submitted
    Assert.assertTrue(isReady);
    final List<DataSegment> segments = Lists.newArrayList();
    mergeTask.run(new TaskToolbox(null, null, new TaskActionClient() {

        @Override
        public <RetType> RetType submit(TaskAction<RetType> taskAction) throws IOException {
            if (taskAction instanceof LockListAction) {
                Assert.assertNotNull("taskLock should be acquired before list", taskLock);
                return (RetType) Arrays.asList(taskLock);
            }
            if (taskAction instanceof SegmentListUsedAction) {
                List<DataSegment> segments = ImmutableList.of(DataSegment.builder().dataSource(mergeTask.getDataSource()).interval(new Interval("2010-01-01/PT1H")).version("oldVersion").shardSpec(new LinearShardSpec(0)).build(), DataSegment.builder().dataSource(mergeTask.getDataSource()).interval(new Interval("2010-01-01/PT1H")).version("oldVersion").shardSpec(new LinearShardSpec(0)).build(), DataSegment.builder().dataSource(mergeTask.getDataSource()).interval(new Interval("2010-01-01/PT2H")).version("oldVersion").shardSpec(new LinearShardSpec(0)).build());
                return (RetType) segments;
            }
            if (taskAction instanceof SegmentInsertAction) {
                publishCountDown.countDown();
                return null;
            }
            return null;
        }
    }, new NoopServiceEmitter(), new DataSegmentPusher() {

        @Deprecated
        @Override
        public String getPathForHadoop(String dataSource) {
            return getPathForHadoop();
        }

        @Override
        public String getPathForHadoop() {
            return null;
        }

        @Override
        public DataSegment push(File file, DataSegment segment) throws IOException {
            // the merged segment is pushed to storage
            segments.add(segment);
            return segment;
        }
    }, null, null, null, null, null, null, null, null, new SegmentLoader() {

        @Override
        public boolean isSegmentLoaded(DataSegment segment) throws SegmentLoadingException {
            return false;
        }

        @Override
        public Segment getSegment(DataSegment segment) throws SegmentLoadingException {
            return null;
        }

        @Override
        public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException {
            // dummy file to represent the downloaded segment's dir
            return new File("" + segment.getShardSpec().getPartitionNum());
        }

        @Override
        public void cleanup(DataSegment segment) throws SegmentLoadingException {
        }
    }, jsonMapper, temporaryFolder.newFolder(), EasyMock.createMock(IndexMerger.class), indexIO, null, null, EasyMock.createMock(IndexMergerV9.class)));
    return segments;
}

Example 14 with Interval

use of org.joda.time.Interval in project druid by druid-io.

the class TaskSerdeTest method testIndexTaskSerde.

@Test
public void testIndexTaskSerde() throws Exception {
    final IndexTask task = new IndexTask(null, null, new IndexTask.IndexIngestionSpec(new DataSchema("foo", null, new AggregatorFactory[] { new DoubleSumAggregatorFactory("met", "met") }, new UniformGranularitySpec(Granularities.DAY, null, ImmutableList.of(new Interval("2010-01-01/P2D"))), jsonMapper), new IndexTask.IndexIOConfig(new LocalFirehoseFactory(new File("lol"), "rofl", null), true, true), new IndexTask.IndexTuningConfig(10000, 10, 9999, null, indexSpec, 3, true, true, true)), null, jsonMapper);
    final String json = jsonMapper.writeValueAsString(task);
    // Just want to run the clock a bit to make sure the task id doesn't change
    Thread.sleep(100);
    final IndexTask task2 = (IndexTask) jsonMapper.readValue(json, Task.class);
    Assert.assertEquals("foo", task.getDataSource());
    Assert.assertEquals(task.getId(), task2.getId());
    Assert.assertEquals(task.getGroupId(), task2.getGroupId());
    Assert.assertEquals(task.getDataSource(), task2.getDataSource());
    IndexTask.IndexIOConfig taskIoConfig = task.getIngestionSchema().getIOConfig();
    IndexTask.IndexIOConfig task2IoConfig = task2.getIngestionSchema().getIOConfig();
    Assert.assertTrue(taskIoConfig.getFirehoseFactory() instanceof LocalFirehoseFactory);
    Assert.assertTrue(task2IoConfig.getFirehoseFactory() instanceof LocalFirehoseFactory);
    Assert.assertEquals(taskIoConfig.isAppendToExisting(), task2IoConfig.isAppendToExisting());
    Assert.assertEquals(taskIoConfig.isSkipFirehoseCaching(), task2IoConfig.isSkipFirehoseCaching());
    IndexTask.IndexTuningConfig taskTuningConfig = task.getIngestionSchema().getTuningConfig();
    IndexTask.IndexTuningConfig task2TuningConfig = task2.getIngestionSchema().getTuningConfig();
    Assert.assertEquals(taskTuningConfig.getBasePersistDirectory(), task2TuningConfig.getBasePersistDirectory());
    Assert.assertEquals(taskTuningConfig.getIndexSpec(), task2TuningConfig.getIndexSpec());
    Assert.assertEquals(taskTuningConfig.getIntermediatePersistPeriod(), task2TuningConfig.getIntermediatePersistPeriod());
    Assert.assertEquals(taskTuningConfig.getMaxPendingPersists(), task2TuningConfig.getMaxPendingPersists());
    Assert.assertEquals(taskTuningConfig.getMaxRowsInMemory(), task2TuningConfig.getMaxRowsInMemory());
    Assert.assertEquals(taskTuningConfig.getNumShards(), task2TuningConfig.getNumShards());
    Assert.assertEquals(taskTuningConfig.getTargetPartitionSize(), task2TuningConfig.getTargetPartitionSize());
    Assert.assertEquals(taskTuningConfig.isBuildV9Directly(), task2TuningConfig.isBuildV9Directly());
    Assert.assertEquals(taskTuningConfig.isForceExtendableShardSpecs(), task2TuningConfig.isForceExtendableShardSpecs());
    Assert.assertEquals(taskTuningConfig.isReportParseExceptions(), task2TuningConfig.isReportParseExceptions());
}

Example 15 with Interval

use of org.joda.time.Interval in project druid by druid-io.

the class RemoteTaskActionClientTest method setUp.

@Before
public void setUp() {
    httpClient = createMock(HttpClient.class);
    selector = createMock(ServerDiscoverySelector.class);
    server = new Server() {

        @Override
        public String getScheme() {
            return "http";
        }

        @Override
        public int getPort() {
            return 8080;
        }

        @Override
        public String getHost() {
            return "localhost";
        }

        @Override
        public String getAddress() {
            return "localhost";
        }
    };
    long now = System.currentTimeMillis();
    result = Arrays.asList(new TaskLock("groupId", "dataSource", new Interval(now - 30 * 1000, now), "version"));
}