Examples with MesosFileChunkObject com.hubspot.mesos.json.MesosFileChunkObject used on opensource projects

Example 6 with MesosFileChunkObject

use of com.hubspot.mesos.json.MesosFileChunkObject in project Singularity by HubSpot.

the class SandboxResource method read.

@GET
@Path("/{taskId}/read")
@ApiOperation("Retrieve part of the contents of a file in a specific task's sandbox.")
public MesosFileChunkObject read(@Auth SingularityUser user, @ApiParam("The task ID of the sandbox to read from") @PathParam("taskId") String taskId, @ApiParam("The path to the file to be read") @QueryParam("path") String path, @ApiParam("Optional string to grep for") @QueryParam("grep") Optional<String> grep, @ApiParam("Byte offset to start reading from") @QueryParam("offset") Optional<Long> offset, @ApiParam("Maximum number of bytes to read") @QueryParam("length") Optional<Long> length) {
    authorizationHelper.checkForAuthorizationByTaskId(taskId, user, SingularityAuthorizationScope.READ);
    final SingularityTaskHistory history = checkHistory(taskId, user);
    checkBadRequest(!Strings.isNullOrEmpty(path), "Must specify 'path'");
    final String slaveHostname = history.getTask().getHostname();
    final String fullPath = new File(history.getDirectory().get(), path).toString();
    try {
        final Optional<MesosFileChunkObject> maybeChunk = sandboxManager.read(slaveHostname, fullPath, offset, length);
        checkNotFound(maybeChunk.isPresent(), "File %s does not exist for task ID %s", fullPath, taskId);
        if (grep.isPresent() && !Strings.isNullOrEmpty(grep.get())) {
            final Pattern grepPattern = Pattern.compile(grep.get());
            final StringBuilder strBuilder = new StringBuilder(maybeChunk.get().getData().length());
            for (String line : Splitter.on("\n").split(maybeChunk.get().getData())) {
                if (grepPattern.matcher(line).find()) {
                    strBuilder.append(line);
                    strBuilder.append("\n");
                }
            }
            return new MesosFileChunkObject(strBuilder.toString(), maybeChunk.get().getOffset(), Optional.of(maybeChunk.get().getOffset() + maybeChunk.get().getData().length()));
        }
        return maybeChunk.get();
    } catch (SlaveNotFoundException snfe) {
        throw notFound("Slave @ %s was not found, it is probably offline", slaveHostname);
    }
}

Example 7 with MesosFileChunkObject

use of com.hubspot.mesos.json.MesosFileChunkObject in project Singularity by HubSpot.

the class SandboxManagerTest method testInvalidUtf8WithLastTwoBytes.

@Test
public void testInvalidUtf8WithLastTwoBytes() throws IOException {
    // data contains last two bytes of a fire character and a ☃ character
    byte[] bytes = toBytes(JSON_START, SECOND_BALLOON_BYTE, THIRD_BALLOON_BYTE, SNOWMAN_UTF8_BYTES, JSON_END);
    MesosFileChunkObject chunk = sandboxManager.parseResponseBody(response(bytes));
    // the partial fire should be dropped and the offset should be advanced by two bytes
    assertThat(chunk.getData()).isEqualTo(SNOWMAN);
    assertThat(chunk.getOffset()).isEqualTo(DEFAULT_OFFSET + 2);
}

Example 8 with MesosFileChunkObject

use of com.hubspot.mesos.json.MesosFileChunkObject in project Singularity by HubSpot.

the class SandboxManagerTest method testInvalidUtf8WithOneByteOfThreeByteCharacter.

@Test
public void testInvalidUtf8WithOneByteOfThreeByteCharacter() throws IOException {
    // data contains a ☃ character and the first byte of another ☃ character
    byte[] bytes = toBytes(JSON_START, SNOWMAN_UTF8_BYTES, FIRST_SNOWMAN_BYTE, JSON_END);
    MesosFileChunkObject chunk = sandboxManager.parseResponseBody(response(bytes));
    // the partial ☃ should be dropped
    assertThat(chunk.getData()).isEqualTo(SNOWMAN);
    assertThat(chunk.getOffset()).isEqualTo(DEFAULT_OFFSET);
}

Example 9 with MesosFileChunkObject

use of com.hubspot.mesos.json.MesosFileChunkObject in project Singularity by HubSpot.

the class SandboxManagerTest method testInvalidUtf8WithTwoBytesOfThreeByteCharacter.

@Test
public void testInvalidUtf8WithTwoBytesOfThreeByteCharacter() throws IOException {
    // data contains a ☃ character and the first two bytes of another ☃ character
    byte[] bytes = toBytes(JSON_START, SNOWMAN_UTF8_BYTES, FIRST_SNOWMAN_BYTE, SECOND_SNOWMAN_BYTE, JSON_END);
    MesosFileChunkObject chunk = sandboxManager.parseResponseBody(response(bytes));
    // the partial ☃ should be dropped
    assertThat(chunk.getData()).isEqualTo(SNOWMAN);
    assertThat(chunk.getOffset()).isEqualTo(DEFAULT_OFFSET);
}

Example 10 with MesosFileChunkObject

use of com.hubspot.mesos.json.MesosFileChunkObject in project Singularity by HubSpot.

the class SandboxManager method parseResponseBody.

/**
 * This method will first sanitize the input by replacing invalid UTF8 characters with \ufffd (Unicode's "REPLACEMENT CHARACTER")
 * before sending it to Jackson for parsing. We then strip the replacement characters characters from the beginning and end of the string
 * and increment the offset field by how many characters were stripped from the beginning.
 */
@VisibleForTesting
MesosFileChunkObject parseResponseBody(Response response) throws IOException {
    // not thread-safe, need to make a new one each time;
    CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPLACE).replaceWith(REPLACEMENT_CHARACTER);
    ByteBuffer responseBuffer = response.getResponseBodyAsByteBuffer();
    Reader sanitizedReader = CharSource.wrap(decoder.decode(responseBuffer)).openStream();
    final MesosFileChunkObject initialChunk = objectMapper.readValue(sanitizedReader, MesosFileChunkObject.class);
    // bail early if no replacement characters
    if (!initialChunk.getData().startsWith(REPLACEMENT_CHARACTER) && !initialChunk.getData().endsWith(REPLACEMENT_CHARACTER)) {
        return initialChunk;
    }
    final String data = initialChunk.getData();
    // if we requested data between two characters, return nothing and advance the offset to the end
    if (data.length() <= 4 && data.replace(REPLACEMENT_CHARACTER, "").length() == 0) {
        return new MesosFileChunkObject("", initialChunk.getOffset() + data.length(), Optional.<Long>absent());
    }
    // trim incomplete character at the beginning of the string
    int startIndex = 0;
    if (data.startsWith(TWO_REPLACEMENT_CHARACTERS)) {
        startIndex = 2;
    } else if (data.startsWith(REPLACEMENT_CHARACTER)) {
        startIndex = 1;
    }
    // trim incomplete character at the end of the string
    int endIndex = data.length();
    if (data.endsWith(TWO_REPLACEMENT_CHARACTERS)) {
        endIndex -= 2;
    } else if (data.endsWith(REPLACEMENT_CHARACTER)) {
        endIndex -= 1;
    }
    return new MesosFileChunkObject(data.substring(startIndex, endIndex), initialChunk.getOffset() + startIndex, Optional.<Long>absent());
}