Examples with NoneShardSpec io.druid.timeline.partition.NoneShardSpec used on opensource projects

Example 1 with NoneShardSpec

use of io.druid.timeline.partition.NoneShardSpec in project druid by druid-io.

the class IndexerSQLMetadataStorageCoordinator method announceHistoricalSegment.

/**
   * Attempts to insert a single segment to the database. If the segment already exists, will do nothing; although,
   * this checking is imperfect and callers must be prepared to retry their entire transaction on exceptions.
   *
   * @return true if the segment was added, false if it already existed
   */
private boolean announceHistoricalSegment(final Handle handle, final DataSegment segment, final boolean used) throws IOException {
    try {
        if (segmentExists(handle, segment)) {
            log.info("Found [%s] in DB, not updating DB", segment.getIdentifier());
            return false;
        }
        // SELECT -> INSERT can fail due to races; callers must be prepared to retry.
        // Avoiding ON DUPLICATE KEY since it's not portable.
        // Avoiding try/catch since it may cause inadvertent transaction-splitting.
        handle.createStatement(String.format("INSERT INTO %1$s (id, dataSource, created_date, start, %2$send%2$s, partitioned, version, used, payload) " + "VALUES (:id, :dataSource, :created_date, :start, :end, :partitioned, :version, :used, :payload)", dbTables.getSegmentsTable(), connector.getQuoteString())).bind("id", segment.getIdentifier()).bind("dataSource", segment.getDataSource()).bind("created_date", new DateTime().toString()).bind("start", segment.getInterval().getStart().toString()).bind("end", segment.getInterval().getEnd().toString()).bind("partitioned", (segment.getShardSpec() instanceof NoneShardSpec) ? false : true).bind("version", segment.getVersion()).bind("used", used).bind("payload", jsonMapper.writeValueAsBytes(segment)).execute();
        log.info("Published segment [%s] to DB", segment.getIdentifier());
    } catch (Exception e) {
        log.error(e, "Exception inserting segment [%s] into DB", segment.getIdentifier());
        throw e;
    }
    return true;
}

Example 2 with NoneShardSpec

use of io.druid.timeline.partition.NoneShardSpec in project druid by druid-io.

the class SameIntervalMergeTaskTest method testRun.

@Test
public void testRun() throws Exception {
    final List<AggregatorFactory> aggregators = ImmutableList.<AggregatorFactory>of(new CountAggregatorFactory("cnt"));
    final SameIntervalMergeTask task = new SameIntervalMergeTask(null, "foo", new Interval("2010-01-01/P1D"), aggregators, true, indexSpec, true, null);
    String newVersion = "newVersion";
    final List<DataSegment> segments = runTask(task, newVersion);
    // the lock is acquired
    Assert.assertEquals(0, isRedayCountDown.getCount());
    // the merged segment is published
    Assert.assertEquals(0, publishCountDown.getCount());
    // the merged segment is the only element
    Assert.assertEquals(1, segments.size());
    DataSegment mergeSegment = segments.get(0);
    Assert.assertEquals("foo", mergeSegment.getDataSource());
    Assert.assertEquals(newVersion, mergeSegment.getVersion());
    // the merged segment's interval is within the requested interval
    Assert.assertTrue(new Interval("2010-01-01/P1D").contains(mergeSegment.getInterval()));
    // the merged segment should be NoneShardSpec
    Assert.assertTrue(mergeSegment.getShardSpec() instanceof NoneShardSpec);
}

Example 3 with NoneShardSpec

use of io.druid.timeline.partition.NoneShardSpec in project druid by druid-io.

the class SchemalessIndexTest method makeAppendedMMappedIndex.

private static QueryableIndex makeAppendedMMappedIndex(Iterable<Pair<String, AggregatorFactory[]>> files, final List<Interval> intervals) {
    try {
        File tmpFile = File.createTempFile("yay", "boo");
        tmpFile.delete();
        File mergedFile = new File(tmpFile, "merged");
        mergedFile.mkdirs();
        mergedFile.deleteOnExit();
        List<File> filesToMap = makeFilesToMap(tmpFile, files);
        VersionedIntervalTimeline<Integer, File> timeline = new VersionedIntervalTimeline<Integer, File>(Ordering.natural().nullsFirst());
        ShardSpec noneShardSpec = NoneShardSpec.instance();
        for (int i = 0; i < intervals.size(); i++) {
            timeline.add(intervals.get(i), i, noneShardSpec.createChunk(filesToMap.get(i)));
        }
        final List<IndexableAdapter> adapters = Lists.newArrayList(Iterables.concat(// TimelineObjectHolder is actually an iterable of iterable of indexable adapters
        Iterables.transform(timeline.lookup(new Interval("1000-01-01/3000-01-01")), new Function<TimelineObjectHolder<Integer, File>, Iterable<IndexableAdapter>>() {

            @Override
            public Iterable<IndexableAdapter> apply(final TimelineObjectHolder<Integer, File> timelineObjectHolder) {
                return Iterables.transform(timelineObjectHolder.getObject(), // Each chunk can be used to build the actual IndexableAdapter
                new Function<PartitionChunk<File>, IndexableAdapter>() {

                    @Override
                    public IndexableAdapter apply(PartitionChunk<File> chunk) {
                        try {
                            return new RowboatFilteringIndexAdapter(new QueryableIndexIndexableAdapter(INDEX_IO.loadIndex(chunk.getObject())), new Predicate<Rowboat>() {

                                @Override
                                public boolean apply(Rowboat input) {
                                    return timelineObjectHolder.getInterval().contains(input.getTimestamp());
                                }
                            });
                        } catch (IOException e) {
                            throw Throwables.propagate(e);
                        }
                    }
                });
            }
        })));
        return INDEX_IO.loadIndex(INDEX_MERGER.append(adapters, null, mergedFile, indexSpec));
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }
}

Example 4 with NoneShardSpec

use of io.druid.timeline.partition.NoneShardSpec in project druid by druid-io.

the class GoogleDataSegmentPusherTest method testPush.

@Test
public void testPush() throws Exception {
    // Create a mock segment on disk
    File tmp = tempFolder.newFile("version.bin");
    final byte[] data = new byte[] { 0x0, 0x0, 0x0, 0x1 };
    Files.write(data, tmp);
    final long size = data.length;
    DataSegment segmentToPush = new DataSegment("foo", new Interval("2015/2016"), "0", Maps.<String, Object>newHashMap(), Lists.<String>newArrayList(), Lists.<String>newArrayList(), new NoneShardSpec(), 0, size);
    GoogleDataSegmentPusher pusher = createMockBuilder(GoogleDataSegmentPusher.class).withConstructor(storage, googleAccountConfig, jsonMapper).addMockedMethod("insert", File.class, String.class, String.class).createMock();
    final String storageDir = DataSegmentPusherUtil.getStorageDir(segmentToPush);
    final String indexPath = prefix + "/" + storageDir + "/" + "index.zip";
    final String descriptorPath = prefix + "/" + storageDir + "/" + "descriptor.json";
    pusher.insert(EasyMock.anyObject(File.class), EasyMock.eq("application/zip"), EasyMock.eq(indexPath));
    expectLastCall();
    pusher.insert(EasyMock.anyObject(File.class), EasyMock.eq("application/json"), EasyMock.eq(descriptorPath));
    expectLastCall();
    replayAll();
    DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush);
    Assert.assertEquals(segmentToPush.getSize(), segment.getSize());
    Assert.assertEquals(segmentToPush, segment);
    Assert.assertEquals(ImmutableMap.of("type", GoogleStorageDruidModule.SCHEME, "bucket", bucket, "path", indexPath), segment.getLoadSpec());
    verifyAll();
}