Examples with SymbolToken com.amazon.ion.SymbolToken used on opensource projects

Example 51 with SymbolToken

use of com.amazon.ion.SymbolToken in project ion-java by amzn.

the class RawValueSpanReaderTest method assertSpan.

private void assertSpan(SpanTester tester) throws Exception {
    IonType type = tester.expectedType;
    // seeks back to the given span
    seekableReader.hoist(tester.span);
    assertEquals(type, reader.next());
    Object expected = tester.expected;
    if (reader.isNullValue()) {
        assertEquals(expected, null);
    } else {
        OffsetSpan valueSpan = (OffsetSpan) spanProvider.valueSpan();
        switch(type) {
            case BOOL:
                assertEquals(expected, reader.booleanValue());
                break;
            case INT:
                assertEquals(expected, reader.bigIntegerValue());
                break;
            case FLOAT:
                assertEquals(expected, reader.doubleValue());
                break;
            case DECIMAL:
                assertEquals(expected, reader.bigDecimalValue());
                break;
            case TIMESTAMP:
                assertEquals(expected, reader.timestampValue());
                break;
            case STRING:
                // A common use case will be to pass strings along without
                // decoding. This tests that case.
                assertArrayEquals(((String) expected).getBytes("UTF-8"), valueBytes(valueSpan));
                assertEquals(expected, reader.stringValue());
                break;
            case SYMBOL:
                // SymbolTokenImpl does not override .equals
                SymbolToken expectedToken = (SymbolToken) expected;
                SymbolToken actualToken = reader.symbolValue();
                assertEquals(expectedToken.getSid(), actualToken.getSid());
                assertEquals(expectedToken.getText(), actualToken.getText());
                break;
            case BLOB:
            case CLOB:
                assertArrayEquals((byte[]) expected, reader.newBytes());
                break;
            case STRUCT:
            case LIST:
            case SEXP:
                reader.stepIn();
                if (reader.next() != null) {
                    // The start position of the container's value span should
                    // be the same as the start position of its first element's
                    // seekable span.
                    long expectedValueStart = valueSpan.getStartOffset();
                    long expectedValueEnd = ((OffsetSpan) seekableReader.currentSpan()).getStartOffset();
                    // skips any nop pad to get at the actual value start
                    expectedValueStart += countNopPad((int) expectedValueStart);
                    if (reader.isInStruct()) {
                        // In structs, however, value spans will start before
                        // the first value's field name SID (VarUInt - 7
                        // bits per byte, hence division by 0x80).
                        expectedValueStart += (reader.getFieldNameSymbol().getSid() / 0x80) + 1;
                    }
                    assertEquals(expectedValueStart, expectedValueEnd);
                }
                reader.stepOut();
                break;
            default:
                throw new IllegalStateException("unexpected type: " + type);
        }
        assertArrayEquals(tester.expectedBytes, valueBytes(valueSpan));
        // All spans over the same value, no matter where they started, should finish at
        // the same position.
        assertEquals(((OffsetSpan) tester.span).getFinishOffset(), valueSpan.getFinishOffset());
    }
}

Example 52 with SymbolToken

use of com.amazon.ion.SymbolToken in project ion-java by amzn.

the class SharedSymbolTableTest method testInternKnownText.

// -------------------------------------------------------------------------
// intern()
@Test
public void testInternKnownText() {
    assertNotSame(A, OTHER_A);
    SymbolTable st = makeAbcTable();
    SymbolToken tok = st.intern(OTHER_A);
    assertSame(A, tok.getText());
    assertEquals(st.getImportedMaxId() + 1, tok.getSid());
}

Example 53 with SymbolToken

use of com.amazon.ion.SymbolToken in project ion-java by amzn.

the class SymbolTableTest method testLocalSymbolTableAppendEmptyList.

@Test
public void testLocalSymbolTableAppendEmptyList() {
    String original = LocalSymbolTablePrefix + "{" + "  symbols:[ \"s1\"]" + "}\n";
    String appended = original + LocalSymbolTablePrefix + "{" + "  imports:" + ION_SYMBOL_TABLE + "," + "  symbols:[]" + "}\n";
    SymbolTable originalSymbolTable = oneValue(original + "null").getSymbolTable();
    SymbolTable appendedSymbolTable = oneValue(appended + "null").getSymbolTable();
    SymbolToken originalSymbol = originalSymbolTable.find("s1");
    SymbolToken appendedSymbol = appendedSymbolTable.find("s1");
    assertEquals(originalSymbol.getSid(), appendedSymbol.getSid());
}

Example 54 with SymbolToken

use of com.amazon.ion.SymbolToken in project ion-java by amzn.

the class LocalSymbolTable method readOneImport.

/**
 * Returns a {@link SymbolTable} representation of a single import
 * declaration from the passed-in reader and catalog.
 *
 * @return
 *          symbol table representation of the import; null if the import
 *          declaration is malformed
 */
private static SymbolTable readOneImport(IonReader ionRep, IonCatalog catalog) {
    assert (ionRep.getType() == IonType.STRUCT);
    String name = null;
    int version = -1;
    int maxid = -1;
    ionRep.stepIn();
    IonType t;
    while ((t = ionRep.next()) != null) {
        if (ionRep.isNullValue())
            continue;
        SymbolToken symTok = ionRep.getFieldNameSymbol();
        int field_id = symTok.getSid();
        if (field_id == UNKNOWN_SYMBOL_ID) {
            // this is a user defined reader or a pure DOM
            // we fall back to text here
            final String fieldName = ionRep.getFieldName();
            field_id = getSidForSymbolTableField(fieldName);
        }
        switch(field_id) {
            case NAME_SID:
                if (t == IonType.STRING) {
                    name = ionRep.stringValue();
                }
                break;
            case VERSION_SID:
                if (t == IonType.INT) {
                    version = ionRep.intValue();
                }
                break;
            case MAX_ID_SID:
                if (t == IonType.INT) {
                    maxid = ionRep.intValue();
                }
                break;
            default:
                // we just ignore anything else as "open content"
                break;
        }
    }
    ionRep.stepOut();
    // Ignore import clauses with malformed name field.
    if (name == null || name.length() == 0 || name.equals(ION)) {
        return null;
    }
    if (version < 1) {
        version = 1;
    }
    SymbolTable itab = null;
    if (catalog != null) {
        itab = catalog.getTable(name, version);
    }
    if (maxid < 0) {
        if (itab == null || version != itab.getVersion()) {
            String message = "Import of shared table " + IonTextUtils.printString(name) + " lacks a valid max_id field, but an exact match was not" + " found in the catalog";
            if (itab != null) {
                message += " (found version " + itab.getVersion() + ")";
            }
            // TODO custom exception
            throw new IonException(message);
        }
        // Exact match is found, but max_id is undefined in import
        // declaration, set max_id to largest sid of shared symtab
        maxid = itab.getMaxId();
    }
    if (itab == null) {
        assert maxid >= 0;
        // Construct substitute table with max_id undefined symbols
        itab = new SubstituteSymbolTable(name, version, maxid);
    } else if (itab.getVersion() != version || itab.getMaxId() != maxid) {
        // A match was found BUT specs are not an exact match
        // Construct a substitute with correct specs, containing the
        // original import table that was found
        itab = new SubstituteSymbolTable(itab, version, maxid);
    }
    return itab;
}

Example 55 with SymbolToken

use of com.amazon.ion.SymbolToken in project ion-java by amzn.

the class LocalSymbolTableImports method find.

/**
 * Finds a symbol already interned by an import, returning the lowest
 * known SID.
 * <p>
 * This method will not necessarily return the same instance given the
 * same input.
 *
 * @param text the symbol text to find
 *
 * @return
 *          the interned symbol (with both text and SID), or {@code null}
 *          if it's not defined by an imported table
 */
SymbolToken find(String text) {
    for (int i = 0; i < myImports.length; i++) {
        SymbolTable importedTable = myImports[i];
        SymbolToken tok = importedTable.find(text);
        if (tok != null) {
            int sid = tok.getSid() + myBaseSids[i];
            // Use interned instance
            text = tok.getText();
            assert text != null;
            return new SymbolTokenImpl(text, sid);
        }
    }
    return null;
}