Examples with IndexSpec io.druid.segment.IndexSpec used on opensource projects

Example 16 with IndexSpec

use of io.druid.segment.IndexSpec in project druid by druid-io.

the class HadoopConverterJobTest method testSimpleJob.

@Test
public void testSimpleJob() throws IOException, InterruptedException {
    final SQLMetadataSegmentManager manager = new SQLMetadataSegmentManager(HadoopDruidConverterConfig.jsonMapper, new Supplier<MetadataSegmentManagerConfig>() {

        @Override
        public MetadataSegmentManagerConfig get() {
            return new MetadataSegmentManagerConfig();
        }
    }, metadataStorageTablesConfigSupplier, connector);
    final List<DataSegment> oldSemgments = getDataSegments(manager);
    final File tmpDir = temporaryFolder.newFolder();
    final HadoopConverterJob converterJob = new HadoopConverterJob(new HadoopDruidConverterConfig(DATASOURCE, interval, new IndexSpec(new RoaringBitmapSerdeFactory(null), CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, CompressionFactory.LongEncodingStrategy.LONGS), oldSemgments, true, tmpDir.toURI(), ImmutableMap.<String, String>of(), null, tmpSegmentDir.toURI().toString()));
    final List<DataSegment> segments = Lists.newArrayList(converterJob.run());
    Assert.assertNotNull("bad result", segments);
    Assert.assertEquals("wrong segment count", 4, segments.size());
    Assert.assertTrue(converterJob.getLoadedBytes() > 0);
    Assert.assertTrue(converterJob.getWrittenBytes() > 0);
    Assert.assertTrue(converterJob.getWrittenBytes() > converterJob.getLoadedBytes());
    Assert.assertEquals(oldSemgments.size(), segments.size());
    final DataSegment segment = segments.get(0);
    Assert.assertTrue(interval.contains(segment.getInterval()));
    Assert.assertTrue(segment.getVersion().endsWith("_converted"));
    Assert.assertTrue(segment.getLoadSpec().get("path").toString().contains("_converted"));
    for (File file : tmpDir.listFiles()) {
        Assert.assertFalse(file.isDirectory());
        Assert.assertTrue(file.isFile());
    }
    final Comparator<DataSegment> segmentComparator = new Comparator<DataSegment>() {

        @Override
        public int compare(DataSegment o1, DataSegment o2) {
            return o1.getIdentifier().compareTo(o2.getIdentifier());
        }
    };
    Collections.sort(oldSemgments, segmentComparator);
    Collections.sort(segments, segmentComparator);
    for (int i = 0; i < oldSemgments.size(); ++i) {
        final DataSegment oldSegment = oldSemgments.get(i);
        final DataSegment newSegment = segments.get(i);
        Assert.assertEquals(oldSegment.getDataSource(), newSegment.getDataSource());
        Assert.assertEquals(oldSegment.getInterval(), newSegment.getInterval());
        Assert.assertEquals(Sets.<String>newHashSet(oldSegment.getMetrics()), Sets.<String>newHashSet(newSegment.getMetrics()));
        Assert.assertEquals(Sets.<String>newHashSet(oldSegment.getDimensions()), Sets.<String>newHashSet(newSegment.getDimensions()));
        Assert.assertEquals(oldSegment.getVersion() + "_converted", newSegment.getVersion());
        Assert.assertTrue(oldSegment.getSize() < newSegment.getSize());
        Assert.assertEquals(oldSegment.getBinaryVersion(), newSegment.getBinaryVersion());
    }
}

Example 17 with IndexSpec

use of io.druid.segment.IndexSpec in project druid by druid-io.

the class HadoopConverterJobTest method testHadoopFailure.

@Test
// This takes a long time due to retries
@Ignore
public void testHadoopFailure() throws IOException, InterruptedException {
    final SQLMetadataSegmentManager manager = new SQLMetadataSegmentManager(HadoopDruidConverterConfig.jsonMapper, new Supplier<MetadataSegmentManagerConfig>() {

        @Override
        public MetadataSegmentManagerConfig get() {
            return new MetadataSegmentManagerConfig();
        }
    }, metadataStorageTablesConfigSupplier, connector);
    final List<DataSegment> oldSemgments = getDataSegments(manager);
    final File tmpDir = temporaryFolder.newFolder();
    final HadoopConverterJob converterJob = new HadoopConverterJob(new HadoopDruidConverterConfig(DATASOURCE, interval, new IndexSpec(new RoaringBitmapSerdeFactory(null), CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, CompressionFactory.LongEncodingStrategy.LONGS), oldSemgments, true, tmpDir.toURI(), ImmutableMap.<String, String>of(), null, tmpSegmentDir.toURI().toString()));
    corrupt(oldSemgments.get(0));
    final List<DataSegment> result = converterJob.run();
    Assert.assertNull("result should be null", result);
    final List<DataSegment> segments = getDataSegments(manager);
    Assert.assertEquals(oldSemgments.size(), segments.size());
    Assert.assertEquals(oldSemgments, segments);
}

Example 18 with IndexSpec

use of io.druid.segment.IndexSpec in project druid by druid-io.

the class HadoopDruidConverterConfigTest method simpleSerDe.

@Test
public void simpleSerDe() throws IOException {
    final HadoopDruidConverterConfig config = new HadoopDruidConverterConfig("datasource", Interval.parse("2000/2010"), new IndexSpec(), ImmutableList.<DataSegment>of(), true, URI.create("file:/dev/null"), ImmutableMap.<String, String>of(), "HIGH", temporaryFolder.newFolder().getAbsolutePath());
    final ObjectMapper mapper = new DefaultObjectMapper();
    mapper.registerSubtypes(HadoopDruidConverterConfig.class);
    final byte[] value = mapper.writeValueAsBytes(config);
    final HadoopDruidConverterConfig config2 = mapper.readValue(value, HadoopDruidConverterConfig.class);
    Assert.assertEquals(mapper.writeValueAsString(config), mapper.writeValueAsString(config2));
}

Example 19 with IndexSpec

use of io.druid.segment.IndexSpec in project druid by druid-io.

the class HadoopTuningConfigTest method testSerde.

@Test
public void testSerde() throws Exception {
    HadoopTuningConfig expected = new HadoopTuningConfig("/tmp/workingpath", "version", null, null, null, 100, true, true, true, true, null, true, true, null, null, null, true, true);
    HadoopTuningConfig actual = jsonReadWriteRead(jsonMapper.writeValueAsString(expected), HadoopTuningConfig.class);
    Assert.assertEquals("/tmp/workingpath", actual.getWorkingPath());
    Assert.assertEquals("version", actual.getVersion());
    Assert.assertNotNull(actual.getPartitionsSpec());
    Assert.assertEquals(ImmutableMap.<DateTime, List<HadoopyShardSpec>>of(), actual.getShardSpecs());
    Assert.assertEquals(new IndexSpec(), actual.getIndexSpec());
    Assert.assertEquals(100, actual.getRowFlushBoundary());
    Assert.assertEquals(true, actual.isLeaveIntermediate());
    Assert.assertEquals(true, actual.isCleanupOnFailure());
    Assert.assertEquals(true, actual.isOverwriteFiles());
    Assert.assertEquals(true, actual.isIgnoreInvalidRows());
    Assert.assertEquals(ImmutableMap.<String, String>of(), actual.getJobProperties());
    Assert.assertEquals(true, actual.isCombineText());
    Assert.assertEquals(true, actual.getUseCombiner());
    Assert.assertEquals(0, actual.getNumBackgroundPersistThreads());
    Assert.assertEquals(true, actual.isForceExtendableShardSpecs());
    Assert.assertEquals(true, actual.isUseExplicitVersion());
}

Example 20 with IndexSpec

use of io.druid.segment.IndexSpec in project druid by druid-io.

the class RealtimePlumber method persistHydrant.

/**
   * Persists the given hydrant and returns the number of rows persisted
   *
   * @param indexToPersist hydrant to persist
   * @param schema         datasource schema
   * @param interval       interval to persist
   *
   * @return the number of rows persisted
   */
protected int persistHydrant(FireHydrant indexToPersist, DataSchema schema, Interval interval, Map<String, Object> metadataElems) {
    synchronized (indexToPersist) {
        if (indexToPersist.hasSwapped()) {
            log.info("DataSource[%s], Interval[%s], Hydrant[%s] already swapped. Ignoring request to persist.", schema.getDataSource(), interval, indexToPersist);
            return 0;
        }
        log.info("DataSource[%s], Interval[%s], Metadata [%s] persisting Hydrant[%s]", schema.getDataSource(), interval, metadataElems, indexToPersist);
        try {
            int numRows = indexToPersist.getIndex().size();
            final IndexSpec indexSpec = config.getIndexSpec();
            indexToPersist.getIndex().getMetadata().putAll(metadataElems);
            final File persistedFile = indexMerger.persist(indexToPersist.getIndex(), interval, new File(computePersistDir(schema, interval), String.valueOf(indexToPersist.getCount())), indexSpec);
            indexToPersist.swapSegment(new QueryableIndexSegment(indexToPersist.getSegment().getIdentifier(), indexIO.loadIndex(persistedFile)));
            return numRows;
        } catch (IOException e) {
            log.makeAlert("dataSource[%s] -- incremental persist failed", schema.getDataSource()).addData("interval", interval).addData("count", indexToPersist.getCount()).emit();
            throw Throwables.propagate(e);
        }
    }
}