Examples with LoadSpec org.apache.druid.segment.loading.LoadSpec used on opensource projects

Example 1 with LoadSpec

use of org.apache.druid.segment.loading.LoadSpec in project druid by druid-io.

the class DeepStorageShuffleClient method fetchSegmentFile.

@Override
public File fetchSegmentFile(File partitionDir, String supervisorTaskId, DeepStoragePartitionLocation location) throws IOException {
    final LoadSpec loadSpec = objectMapper.convertValue(location.getLoadSpec(), LoadSpec.class);
    final File unzippedDir = new File(partitionDir, StringUtils.format("unzipped_%s", location.getSubTaskId()));
    FileUtils.mkdirp(unzippedDir);
    try {
        loadSpec.loadSegment(unzippedDir);
    } catch (SegmentLoadingException e) {
        LOG.error(e, "Failed to load segment");
        throw new IOException(e);
    }
    return unzippedDir;
}

Example 2 with LoadSpec

use of org.apache.druid.segment.loading.LoadSpec in project druid by druid-io.

the class ShuffleDataSegmentPusherTest method testPush.

@Test
public void testPush() throws IOException, SegmentLoadingException {
    final File segmentDir = generateSegmentDir();
    final DataSegment segment = newSegment(Intervals.of("2018/2019"));
    final DataSegment pushed = segmentPusher.push(segmentDir, segment, true);
    Assert.assertEquals(9, pushed.getBinaryVersion().intValue());
    // 10 bytes data + 4 bytes version
    Assert.assertEquals(14, pushed.getSize());
    final File tempDir = temporaryFolder.newFolder();
    if (intermediaryDataManager instanceof LocalIntermediaryDataManager) {
        final Optional<ByteSource> zippedSegment = intermediaryDataManager.findPartitionFile("supervisorTaskId", "subTaskId", segment.getInterval(), segment.getShardSpec().getPartitionNum());
        Assert.assertTrue(zippedSegment.isPresent());
        CompressionUtils.unzip(zippedSegment.get(), tempDir, org.apache.druid.java.util.common.FileUtils.IS_EXCEPTION, false);
    } else if (intermediaryDataManager instanceof DeepStorageIntermediaryDataManager) {
        final LoadSpec loadSpec = mapper.convertValue(pushed.getLoadSpec(), LoadSpec.class);
        Assert.assertTrue(pushed.getLoadSpec().get("path").toString().startsWith(localDeepStore.getAbsolutePath() + "/" + DeepStorageIntermediaryDataManager.SHUFFLE_DATA_DIR_PREFIX));
        loadSpec.loadSegment(tempDir);
    }
    final List<File> unzippedFiles = Arrays.asList(tempDir.listFiles());
    unzippedFiles.sort(Comparator.comparing(File::getName));
    final File dataFile = unzippedFiles.get(0);
    Assert.assertEquals("test", dataFile.getName());
    Assert.assertEquals("test data.", Files.readFirstLine(dataFile, StandardCharsets.UTF_8));
    final File versionFile = unzippedFiles.get(1);
    Assert.assertEquals("version.bin", versionFile.getName());
    Assert.assertArrayEquals(Ints.toByteArray(0x9), Files.toByteArray(versionFile));
}