Examples with MapWriter org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter used on opensource projects

Example 1 with MapWriter

use of org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter in project drill by apache.

the class TestRepeated method listOfList.

//
//  @Test
//  public void repeatedMap() {
//
//    /**
//     * We're going to try to create an object that looks like:
//     *
//     *  {
//     *    a: [
//     *      {x: 1, y: 2}
//     *      {x: 2, y: 1}
//     *    ]
//     *  }
//     *
//     */
//    MapVector v = new MapVector("", allocator);
//    ComplexWriter writer = new ComplexWriterImpl("col", v);
//
//    MapWriter map = writer.rootAsMap();
//
//    map.start();
//    ListWriter list = map.list("a");
//    MapWriter inner = list.map();
//
//    IntHolder holder = new IntHolder();
//    IntWriter xCol = inner.integer("x");
//    IntWriter yCol = inner.integer("y");
//
//    inner.start();
//
//    holder.value = 1;
//    xCol.write(holder);
//    holder.value = 2;
//    yCol.write(holder);
//
//    inner.end();
//
//    inner.start();
//
//    holder.value = 2;
//    xCol.write(holder);
//    holder.value = 1;
//    yCol.write(holder);
//
//    inner.end();
//
//    IntWriter numCol = map.integer("nums");
//    holder.value = 14;
//    numCol.write(holder);
//
//    map.end();
//
//
//    assertTrue(writer.ok());
//
//    System.out.println(v.getAccessor().getObject(0));
//
//  }
@Test
public void listOfList() throws Exception {
    /**
     * We're going to try to create an object that looks like:
     *
     *  {
     *    a: [
     *      [1,2,3],
     *      [2,3,4]
     *    ],
     *    nums: 14,
     *    b: [
     *      { c: 1 },
     *      { c: 2 , x: 15}
     *    ]
     *  }
     *
     */
    final MapVector mapVector = new MapVector("", allocator, null);
    final ComplexWriterImpl writer = new ComplexWriterImpl("col", mapVector);
    writer.allocate();
    {
        final MapWriter map = writer.rootAsMap();
        final ListWriter list = map.list("a");
        list.startList();
        final ListWriter innerList = list.list();
        final IntWriter innerInt = innerList.integer();
        innerList.startList();
        final IntHolder holder = new IntHolder();
        holder.value = 1;
        innerInt.write(holder);
        holder.value = 2;
        innerInt.write(holder);
        holder.value = 3;
        innerInt.write(holder);
        innerList.endList();
        innerList.startList();
        holder.value = 4;
        innerInt.write(holder);
        holder.value = 5;
        innerInt.write(holder);
        innerList.endList();
        list.endList();
        final IntWriter numCol = map.integer("nums");
        holder.value = 14;
        numCol.write(holder);
        final MapWriter repeatedMap = map.list("b").map();
        repeatedMap.start();
        holder.value = 1;
        repeatedMap.integer("c").write(holder);
        repeatedMap.end();
        repeatedMap.start();
        holder.value = 2;
        repeatedMap.integer("c").write(holder);
        final BigIntHolder h = new BigIntHolder();
        h.value = 15;
        repeatedMap.bigInt("x").write(h);
        repeatedMap.end();
        map.end();
    }
    {
        writer.setPosition(1);
        final MapWriter map = writer.rootAsMap();
        final ListWriter list = map.list("a");
        list.startList();
        final ListWriter innerList = list.list();
        final IntWriter innerInt = innerList.integer();
        innerList.startList();
        final IntHolder holder = new IntHolder();
        holder.value = -1;
        innerInt.write(holder);
        holder.value = -2;
        innerInt.write(holder);
        holder.value = -3;
        innerInt.write(holder);
        innerList.endList();
        innerList.startList();
        holder.value = -4;
        innerInt.write(holder);
        holder.value = -5;
        innerInt.write(holder);
        innerList.endList();
        list.endList();
        final IntWriter numCol = map.integer("nums");
        holder.value = -28;
        numCol.write(holder);
        final MapWriter repeatedMap = map.list("b").map();
        repeatedMap.start();
        holder.value = -1;
        repeatedMap.integer("c").write(holder);
        repeatedMap.end();
        repeatedMap.start();
        holder.value = -2;
        repeatedMap.integer("c").write(holder);
        final BigIntHolder h = new BigIntHolder();
        h.value = -30;
        repeatedMap.bigInt("x").write(h);
        repeatedMap.end();
        map.end();
    }
    final ObjectWriter ow = new ObjectMapper().writer().withDefaultPrettyPrinter();
    System.out.println("Map of Object[0]: " + ow.writeValueAsString(mapVector.getAccessor().getObject(0)));
    System.out.println("Map of Object[1]: " + ow.writeValueAsString(mapVector.getAccessor().getObject(1)));
    final ByteArrayOutputStream stream = new ByteArrayOutputStream();
    final JsonWriter jsonWriter = new JsonWriter(stream, true, true);
    final FieldReader reader = mapVector.getChild("col", MapVector.class).getReader();
    reader.setPosition(0);
    jsonWriter.write(reader);
    reader.setPosition(1);
    jsonWriter.write(reader);
    System.out.print("Json Read: ");
    System.out.println(new String(stream.toByteArray(), Charsets.UTF_8));
    writer.close();
}

Example 2 with MapWriter

use of org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter in project drill by apache.

the class HttpdLogRecord method getWildcardWriter.

/**
   * For a configuration like HTTP.URI:request.firstline.uri.query.*, a writer was created with name
   * HTTP.URI:request.firstline.uri.query, we traverse the list of wildcard writers to see which one is the root of the
   * name of the field passed in like HTTP.URI:request.firstline.uri.query.old. This is writer entry that is needed.
   *
   * @param field like HTTP.URI:request.firstline.uri.query.old where 'old' is one of many different parameter names.
   * @return the writer to be used for this field.
   */
private MapWriter getWildcardWriter(final String field) {
    MapWriter writer = startedWildcards.get(field);
    if (writer == null) {
        for (Map.Entry<String, MapWriter> entry : wildcards.entrySet()) {
            final String root = entry.getKey();
            if (field.startsWith(root)) {
                writer = entry.getValue();
                /**
           * In order to save some time, store the cleaned version of the field extension. It is possible it will have
           * unsafe characters in it.
           */
                if (!cleanExtensions.containsKey(field)) {
                    final String extension = field.substring(root.length() + 1, field.length());
                    final String cleanExtension = HttpdParser.drillFormattedFieldName(extension);
                    cleanExtensions.put(field, cleanExtension);
                    LOG.debug("Added extension: field='{}' with cleanExtension='{}'", field, cleanExtension);
                }
                /**
           * We already know we have the writer, but if we have put this writer in the started list, do NOT call start
           * again.
           */
                if (!wildcardWriters.containsKey(root)) {
                    /**
             * Start and store this root map writer for later retrieval.
             */
                    LOG.debug("Starting new wildcard field writer: {}", field);
                    writer.start();
                    startedWildcards.put(field, writer);
                    wildcardWriters.put(root, writer);
                }
                /**
           * Break out of the for loop when we find a root writer that matches the field.
           */
                break;
            }
        }
    }
    return writer;
}

Example 3 with MapWriter

use of org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter in project drill by apache.

the class HttpdLogRecord method finishRecord.

/**
   * Call this method after a record has been parsed. This finished the lifecycle of any maps that were written and
   * removes all the entries for the next record to be able to work.
   */
public void finishRecord() {
    for (MapWriter writer : wildcardWriters.values()) {
        writer.end();
    }
    wildcardWriters.clear();
    startedWildcards.clear();
}

Example 4 with MapWriter

use of org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter in project drill by apache.

the class HttpdParserTest method runTest.

private void runTest(String logFormat, String logLine) throws Exception {
    MapWriter mapWriter = null;
    DrillBuf managedBuffer = null;
    Map<String, String> configuredFields = null;
    HttpdParser parser = new HttpdParser(mapWriter, managedBuffer, logFormat, null, configuredFields);
    parser.parse(logLine);
}

Example 5 with MapWriter

use of org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter in project drill by apache.

the class HttpdLogRecord method setWildcard.

/**
   * This method is referenced and called via reflection. When the parser processes a field like:
   * HTTP.URI:request.firstline.uri.query.* where star is an arbitrary field that the parser found this method will be
   * invoked. <br>
   *
   * @param field name of field
   * @param value value of field
   */
@SuppressWarnings("unused")
public void setWildcard(final String field, final Long value) {
    if (value != null) {
        final MapWriter mapWriter = getWildcardWriter(field);
        LOG.trace("Parsed wildcard field: {}, as long: {}", field, value);
        final BigIntWriter w = mapWriter.bigInt(cleanExtensions.get(field));
        w.writeBigInt(value);
    }
}