Examples with ByteArrayOutputStream org.apache.commons.io.output.ByteArrayOutputStream used on opensource projects

Example 76 with ByteArrayOutputStream

use of org.apache.commons.io.output.ByteArrayOutputStream in project sling by apache.

the class DistributionPackageUtilsTest method testInfoEmptyStreams.

@Test
public void testInfoEmptyStreams() {
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    Map<String, Object> info = new HashMap<String, Object>();
    DistributionPackageUtils.writeInfo(outputStream, info);
    InputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
    Map<String, Object> resultInfo = new HashMap<String, Object>();
    DistributionPackageUtils.readInfo(inputStream, resultInfo);
    assertEquals(info.size(), resultInfo.size());
}

Example 77 with ByteArrayOutputStream

use of org.apache.commons.io.output.ByteArrayOutputStream in project Gargoyle by callakrsos.

the class MonitorsTest method test.

@Test
public final void test() throws Exception {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    Monitors.runStackTool(pid, out);
    System.out.println(out.toString());
}

Example 78 with ByteArrayOutputStream

use of org.apache.commons.io.output.ByteArrayOutputStream in project Anserini by castorini.

the class IndexW2V method indexEmbeddings.

public void indexEmbeddings() throws IOException, InterruptedException {
    LOG.info("Starting indexer...");
    long startTime = System.currentTimeMillis();
    final WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer();
    final IndexWriterConfig config = new IndexWriterConfig(analyzer);
    final IndexWriter writer = new IndexWriter(directory, config);
    BufferedReader bRdr = new BufferedReader(new FileReader(args.input));
    String line = null;
    bRdr.readLine();
    Document document = new Document();
    ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
    int cnt = 0;
    while ((line = bRdr.readLine()) != null) {
        String[] termEmbedding = line.trim().split("\t");
        document.add(new StringField(LuceneDocumentGenerator.FIELD_ID, termEmbedding[0], Field.Store.NO));
        String[] parts = termEmbedding[1].split(" ");
        for (int i = 0; i < parts.length; ++i) {
            byteStream.write(ByteBuffer.allocate(4).putFloat(Float.parseFloat(parts[i])).array());
        }
        document.add(new StoredField(FIELD_BODY, byteStream.toByteArray()));
        byteStream.flush();
        byteStream.reset();
        writer.addDocument(document);
        document.clear();
        cnt++;
        if (cnt % 100000 == 0) {
            LOG.info(cnt + " terms indexed");
        }
    }
    LOG.info(String.format("Total of %s terms added", cnt));
    try {
        writer.commit();
        writer.forceMerge(1);
    } finally {
        try {
            writer.close();
        } catch (IOException e) {
            LOG.error(e);
        }
    }
    LOG.info("Total elapsed time: " + (System.currentTimeMillis() - startTime) + "ms");
}

Example 79 with ByteArrayOutputStream

use of org.apache.commons.io.output.ByteArrayOutputStream in project ddf by codice.

the class AtomTransformer method transform.

@Override
public BinaryContent transform(SourceResponse sourceResponse, Map<String, Serializable> arguments) throws CatalogTransformerException {
    if (sourceResponse == null) {
        throw new CatalogTransformerException("Cannot transform null " + SourceResponse.class.getName());
    }
    Date currentDate = new Date();
    ClassLoader tccl = Thread.currentThread().getContextClassLoader();
    Feed feed = null;
    try {
        Thread.currentThread().setContextClassLoader(AtomTransformer.class.getClassLoader());
        feed = ABDERA.newFeed();
    } finally {
        Thread.currentThread().setContextClassLoader(tccl);
    }
    /*
         * Atom spec text (rfc4287) Sect 4.2.14: "The "atom:title" element is a Text construct that
         * conveys a human- readable title for an entry or feed."
         */
    feed.setTitle(DEFAULT_FEED_TITLE);
    feed.setUpdated(currentDate);
    // TODO Use the same id for the same query
    // one challenge is a query in one site should not have the same feed id
    // as a query in another site probably could factor in ddf.host and port
    // into the algorithm
    feed.setId(URN_UUID + UUID.randomUUID().toString());
    // TODO SELF LINK For the Feed, possible design --> serialize Query into
    // a URL
    /*
         * Atom spec text (rfc4287): "atom:feed elements SHOULD contain one atom:link element with a
         * rel attribute value of self. This is the preferred URI for retrieving Atom Feed Documents
         * representing this Atom feed. "
         */
    feed.addLink("#", Link.REL_SELF);
    if (!StringUtils.isEmpty(SystemInfo.getOrganization())) {
        feed.addAuthor(SystemInfo.getOrganization());
    } else {
        feed.addAuthor(DEFAULT_AUTHOR);
    }
    /*
         * Atom spec text (rfc4287 sect. 4.2.4): "The "atom:generator" element's content identifies
         * the agent used to generate a feed, for debugging and other purposes." Generator is not
         * required in the atom:feed element.
         */
    if (!StringUtils.isEmpty(SystemInfo.getSiteName())) {
        // text is required.
        feed.setGenerator(null, SystemInfo.getVersion(), SystemInfo.getSiteName());
    }
    /*
         * According to http://www.opensearch.org/Specifications/OpenSearch/1.1 specification,
         * totalResults must be a non-negative integer. Requirements: This attribute is optional.
         */
    if (sourceResponse.getHits() > -1) {
        Element hits = feed.addExtension(OpenSearchConstants.TOTAL_RESULTS);
        hits.setText(Long.toString(sourceResponse.getHits()));
    }
    if (sourceResponse.getRequest() != null && sourceResponse.getRequest().getQuery() != null) {
        Element itemsPerPage = feed.addExtension(OpenSearchConstants.ITEMS_PER_PAGE);
        Element startIndex = feed.addExtension(OpenSearchConstants.START_INDEX);
        /*
             * According to http://www.opensearch.org/Specifications/OpenSearch/1.1 specification,
             * itemsPerPage must be a non-negative integer. It is possible that Catalog pageSize is
             * set to a non-negative integer though. When non-negative we will instead we will
             * change it to the number of search results on current page.
             */
        if (sourceResponse.getRequest().getQuery().getPageSize() > -1) {
            itemsPerPage.setText(Integer.toString(sourceResponse.getRequest().getQuery().getPageSize()));
        } else {
            if (sourceResponse.getResults() != null) {
                itemsPerPage.setText(Integer.toString(sourceResponse.getResults().size()));
            }
        }
        startIndex.setText(Integer.toString(sourceResponse.getRequest().getQuery().getStartIndex()));
    }
    for (Result result : sourceResponse.getResults()) {
        Metacard metacard = result.getMetacard();
        if (metacard == null) {
            continue;
        }
        Entry entry = feed.addEntry();
        String sourceName = DEFAULT_SOURCE_ID;
        if (result.getMetacard().getSourceId() != null) {
            sourceName = result.getMetacard().getSourceId();
        }
        Element source = entry.addExtension(new QName(FEDERATION_EXTENSION_NAMESPACE, "resultSource", "fs"));
        /*
             * According to the os-federation.xsd, the resultSource element text has a max length of
             * 16 and is the shortname of the source id. Previously, we were duplicating the names
             * in both positions, but since we truly do not have a shortname for our source ids, I
             * am purposely omitting the shortname text and leaving it as the empty string. The real
             * source id can still be found in the attribute instead.
             */
        source.setAttributeValue(new QName(FEDERATION_EXTENSION_NAMESPACE, "sourceId"), sourceName);
        if (result.getRelevanceScore() != null) {
            Element relevance = entry.addExtension(new QName("http://a9.com/-/opensearch/extensions/relevance/1.0/", "score", "relevance"));
            relevance.setText(result.getRelevanceScore().toString());
        }
        entry.setId(URN_CATALOG_ID + metacard.getId());
        /*
             * Atom spec text (rfc4287): "The "atom:title" element is a Text construct that conveys
             * a human- readable title for an entry or feed."
             */
        entry.setTitle(metacard.getTitle());
        /*
             * Atom spec text (rfc4287): "The "atom:updated" element is a Date construct indicating
             * the most recent instant in time when an entry or feed was modified in a way the
             * publisher considers significant." Therefore, a new Date is used because we are making
             * the entry for the first time.
             */
        if (metacard.getModifiedDate() != null) {
            entry.setUpdated(metacard.getModifiedDate());
        } else {
            entry.setUpdated(currentDate);
        }
        /*
             * Atom spec text (rfc4287): "Typically, atom:published will be associated with the
             * initial creation or first availability of the resource."
             */
        if (metacard.getCreatedDate() != null) {
            entry.setPublished(metacard.getCreatedDate());
        }
        /*
             * For atom:link elements, Atom spec text (rfc4287): "The value "related" signifies that
             * the IRI in the value of the href attribute identifies a resource related to the
             * resource described by the containing element."
             */
        addLink(resourceActionProvider, metacard, entry, Link.REL_RELATED);
        addLink(viewMetacardActionProvider, metacard, entry, Link.REL_ALTERNATE);
        addLink(thumbnailActionProvider, metacard, entry, REL_PREVIEW);
        /*
             * Atom spec text (rfc4287) Sect. 4.2.2.: "The "atom:category" element conveys
             * information about a category associated with an entry or feed. This specification
             * assigns no meaning to the content (if any) of this element."
             */
        if (metacard.getContentTypeName() != null) {
            entry.addCategory(metacard.getContentTypeName());
        }
        for (Position position : getGeoRssPositions(metacard)) {
            GeoHelper.addPosition(entry, position, Encoding.GML);
        }
        BinaryContent binaryContent = null;
        String contentOutput = metacard.getId();
        Type atomContentType = Type.TEXT;
        if (metacardTransformer != null) {
            try {
                binaryContent = metacardTransformer.transform(metacard, new HashMap<>());
            } catch (CatalogTransformerException | RuntimeException e) {
                LOGGER.debug(COULD_NOT_CREATE_XML_CONTENT_MESSAGE, e);
            }
            if (binaryContent != null) {
                try {
                    byte[] xmlBytes = binaryContent.getByteArray();
                    if (xmlBytes != null && xmlBytes.length > 0) {
                        contentOutput = new String(xmlBytes, StandardCharsets.UTF_8);
                        atomContentType = Type.XML;
                    }
                } catch (IOException e) {
                    LOGGER.debug(COULD_NOT_CREATE_XML_CONTENT_MESSAGE, e);
                }
            }
        }
        tccl = Thread.currentThread().getContextClassLoader();
        try {
            Thread.currentThread().setContextClassLoader(AtomTransformer.class.getClassLoader());
            entry.setContent(contentOutput, atomContentType);
        } finally {
            Thread.currentThread().setContextClassLoader(tccl);
        }
    }
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try {
        tccl = Thread.currentThread().getContextClassLoader();
        try {
            Thread.currentThread().setContextClassLoader(AtomTransformer.class.getClassLoader());
            feed.writeTo(baos);
        } finally {
            Thread.currentThread().setContextClassLoader(tccl);
        }
    } catch (IOException e) {
        LOGGER.info("Could not write to output stream.", e);
        throw new CatalogTransformerException("Could not transform into Atom.", e);
    }
    return new BinaryContentImpl(new ByteArrayInputStream(baos.toByteArray()), MIME_TYPE);
}

Example 80 with ByteArrayOutputStream

use of org.apache.commons.io.output.ByteArrayOutputStream in project ddf by codice.

the class SolrProviderTest method testExtensibleMetacards.

@Test
public void testExtensibleMetacards() throws IngestException, UnsupportedQueryException {
    deleteAllIn(provider);
    String brandNewField1 = "description";
    String brandNewFieldValue1 = "myDescription";
    /* BASIC STRINGS */
    Set<AttributeDescriptor> descriptors = new HashSet<AttributeDescriptor>();
    descriptors.add(new AttributeDescriptorImpl(Metacard.ID, true, true, true, false, BasicTypes.STRING_TYPE));
    descriptors.add(new AttributeDescriptorImpl(brandNewField1, true, true, true, false, BasicTypes.STRING_TYPE));
    MetacardTypeImpl mType = new MetacardTypeImpl("custom1", descriptors);
    MetacardImpl customMetacard = new MetacardImpl(mType);
    // customMetacard.setAttribute("id", "44567880");
    customMetacard.setAttribute(brandNewField1, brandNewFieldValue1);
    create(customMetacard);
    // search id
    Query query = new QueryImpl(filterBuilder.attribute("id").like().text("*"));
    QueryRequest request = new QueryRequestImpl(query);
    SourceResponse response = provider.query(request);
    assertEquals(1, response.getResults().size());
    assertEquals("custom1", response.getResults().get(0).getMetacard().getMetacardType().getName());
    assertThat(response.getResults().get(0).getMetacard().getMetacardType().getAttributeDescriptors(), equalTo(descriptors));
    // search title - *
    query = new QueryImpl(filterBuilder.attribute(brandNewField1).like().text("*"));
    request = new QueryRequestImpl(query);
    response = provider.query(request);
    assertEquals(1, response.getResults().size());
    assertEquals("custom1", response.getResults().get(0).getMetacard().getMetacardType().getName());
    assertThat(response.getResults().get(0).getMetacard().getMetacardType().getAttributeDescriptors(), equalTo(descriptors));
    // search title - exact
    query = new QueryImpl(filterBuilder.attribute(brandNewField1).equalTo().text(brandNewFieldValue1));
    request = new QueryRequestImpl(query);
    response = provider.query(request);
    assertEquals(1, response.getResults().size());
    // search negative
    query = new QueryImpl(filterBuilder.attribute(brandNewField1).like().text("no"));
    request = new QueryRequestImpl(query);
    response = provider.query(request);
    assertEquals(0, response.getResults().size());
    // NEW TYPE
    String brandNewXmlField1 = "author";
    String brandNewXmlFieldValue1 = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n" + "<author>john doe</author>";
    descriptors = new HashSet<AttributeDescriptor>();
    descriptors.add(new AttributeDescriptorImpl(Metacard.ID, true, true, true, false, BasicTypes.STRING_TYPE));
    descriptors.add(new AttributeDescriptorImpl(brandNewXmlField1, true, true, true, false, BasicTypes.XML_TYPE));
    mType = new MetacardTypeImpl("34ga$^TGHfg:/", descriptors);
    customMetacard = new MetacardImpl(mType);
    // customMetacard.setAttribute(Metacard.ID, "44567880");
    customMetacard.setAttribute(brandNewXmlField1, brandNewXmlFieldValue1);
    create(customMetacard);
    // Two Ids
    query = new QueryImpl(filterBuilder.attribute(Metacard.ID).like().text("*"));
    request = new QueryRequestImpl(query);
    response = provider.query(request);
    assertEquals(2, response.getResults().size());
    // search xml
    query = new QueryImpl(filterBuilder.attribute(brandNewXmlField1).like().caseSensitiveText("doe"));
    request = new QueryRequestImpl(query);
    response = provider.query(request);
    assertEquals(1, response.getResults().size());
    assertEquals("34ga$^TGHfg:/", response.getResults().get(0).getMetacard().getMetacardType().getName());
    assertThat(response.getResults().get(0).getMetacard().getMetacardType().getAttributeDescriptors(), equalTo(descriptors));
    // search xml - negative
    query = new QueryImpl(filterBuilder.attribute(brandNewXmlField1).like().text("author"));
    request = new QueryRequestImpl(query);
    response = provider.query(request);
    assertEquals(0, response.getResults().size());
    // EVERYTHING ELSE type
    String doubleField = "hertz";
    double doubleFieldValue = 16065.435;
    String floatField = "inches";
    float floatFieldValue = 4.435f;
    String intField = "count";
    int intFieldValue = 4;
    String longField = "milliseconds";
    long longFieldValue = 987654322111L;
    String byteField = "bytes";
    byte[] byteFieldValue = { 86 };
    String booleanField = "expected";
    boolean booleanFieldValue = true;
    String dateField = "lost";
    Date dateFieldValue = new Date();
    String geoField = "geo";
    String geoFieldValue = GULF_OF_GUINEA_POINT_WKT;
    String shortField = "daysOfTheWeek";
    short shortFieldValue = 1;
    String objectField = "payload";
    BevelBorder objectFieldValue = new BevelBorder(BevelBorder.RAISED);
    descriptors = new HashSet<AttributeDescriptor>();
    descriptors.add(new AttributeDescriptorImpl(Metacard.ID, true, true, true, false, BasicTypes.STRING_TYPE));
    descriptors.add(new AttributeDescriptorImpl(doubleField, true, true, false, false, BasicTypes.DOUBLE_TYPE));
    descriptors.add(new AttributeDescriptorImpl(floatField, true, true, false, false, BasicTypes.FLOAT_TYPE));
    descriptors.add(new AttributeDescriptorImpl(intField, true, true, false, false, BasicTypes.INTEGER_TYPE));
    descriptors.add(new AttributeDescriptorImpl(longField, true, true, false, false, BasicTypes.LONG_TYPE));
    descriptors.add(new AttributeDescriptorImpl(byteField, false, true, false, false, BasicTypes.BINARY_TYPE));
    descriptors.add(new AttributeDescriptorImpl(booleanField, true, true, false, false, BasicTypes.BOOLEAN_TYPE));
    descriptors.add(new AttributeDescriptorImpl(dateField, true, true, false, false, BasicTypes.DATE_TYPE));
    descriptors.add(new AttributeDescriptorImpl(geoField, true, true, false, false, BasicTypes.GEO_TYPE));
    descriptors.add(new AttributeDescriptorImpl(shortField, true, true, false, false, BasicTypes.SHORT_TYPE));
    descriptors.add(new AttributeDescriptorImpl(objectField, false, true, false, false, BasicTypes.OBJECT_TYPE));
    mType = new MetacardTypeImpl("numbersMT", descriptors);
    customMetacard = new MetacardImpl(mType);
    // customMetacard.setAttribute(Metacard.ID, "245gasg324");
    customMetacard.setAttribute(doubleField, doubleFieldValue);
    customMetacard.setAttribute(floatField, floatFieldValue);
    customMetacard.setAttribute(intField, intFieldValue);
    customMetacard.setAttribute(longField, longFieldValue);
    customMetacard.setAttribute(byteField, byteFieldValue);
    customMetacard.setAttribute(booleanField, booleanFieldValue);
    customMetacard.setAttribute(dateField, dateFieldValue);
    customMetacard.setAttribute(geoField, geoFieldValue);
    customMetacard.setAttribute(shortField, shortFieldValue);
    customMetacard.setAttribute(objectField, objectFieldValue);
    create(customMetacard);
    // Three Ids
    query = new QueryImpl(filterBuilder.attribute(Metacard.ID).like().text("*"));
    request = new QueryRequestImpl(query);
    response = provider.query(request);
    assertEquals(3, response.getResults().size());
    // search double
    query = new QueryImpl(filterBuilder.attribute(doubleField).greaterThan().number(doubleFieldValue - 1.0));
    request = new QueryRequestImpl(query);
    response = provider.query(request);
    assertEquals(1, response.getResults().size());
    // search int
    query = new QueryImpl(filterBuilder.attribute(intField).greaterThan().number(intFieldValue - 1));
    request = new QueryRequestImpl(query);
    response = provider.query(request);
    assertEquals(1, response.getResults().size());
    Metacard resultMetacard = response.getResults().get(0).getMetacard();
    assertThat(resultMetacard.getAttribute(Metacard.ID), notNullValue());
    assertThat((Double) (resultMetacard.getAttribute(doubleField).getValue()), equalTo(doubleFieldValue));
    assertThat((Integer) (resultMetacard.getAttribute(intField).getValue()), equalTo(intFieldValue));
    assertThat((Float) (resultMetacard.getAttribute(floatField).getValue()), equalTo(floatFieldValue));
    assertThat((Long) (resultMetacard.getAttribute(longField).getValue()), equalTo(longFieldValue));
    assertThat((byte[]) (resultMetacard.getAttribute(byteField).getValue()), equalTo(byteFieldValue));
    assertThat((Boolean) (resultMetacard.getAttribute(booleanField).getValue()), equalTo(booleanFieldValue));
    assertThat((Date) (resultMetacard.getAttribute(dateField).getValue()), equalTo(dateFieldValue));
    assertThat((String) (resultMetacard.getAttribute(geoField).getValue()), equalTo(geoFieldValue));
    assertThat((Short) (resultMetacard.getAttribute(shortField).getValue()), equalTo(shortFieldValue));
    assertThat((BevelBorder) (resultMetacard.getAttribute(objectField).getValue()), instanceOf(BevelBorder.class));
    /*
         * Going to use the XMLEncoder. If it writes out the objects the same way in xml, then they
         * are the same.
         */
    ByteArrayOutputStream beveledBytesStreamFromSolr = new ByteArrayOutputStream();
    XMLEncoder solrXMLEncoder = new XMLEncoder(new BufferedOutputStream(beveledBytesStreamFromSolr));
    solrXMLEncoder.writeObject((resultMetacard.getAttribute(objectField).getValue()));
    solrXMLEncoder.close();
    ByteArrayOutputStream beveledBytesStreamOriginal = new ByteArrayOutputStream();
    XMLEncoder currendEncoder = new XMLEncoder(new BufferedOutputStream(beveledBytesStreamOriginal));
    currendEncoder.writeObject(objectFieldValue);
    currendEncoder.close();
    assertThat(beveledBytesStreamFromSolr.toByteArray(), equalTo(beveledBytesStreamOriginal.toByteArray()));
    // search short
    query = new QueryImpl(filterBuilder.attribute(shortField).greaterThanOrEqualTo().number(shortFieldValue));
    request = new QueryRequestImpl(query);
    response = provider.query(request);
    assertEquals(1, response.getResults().size());
    resultMetacard = response.getResults().get(0).getMetacard();
}