Examples with JournalEntryWriter com.questdb.store.JournalEntryWriter used on opensource projects

Example 1 with JournalEntryWriter

use of com.questdb.store.JournalEntryWriter in project questdb by bluestreak01.

the class JoinQueryTest method generateJoinData.

private static void generateJoinData() throws JournalException, NumericException {
    try (JournalWriter customers = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("customers").$int("customerId").$str("customerName").$str("contactName").$str("address").$str("city").$str("postalCode").$sym("country").$ts())) {
        try (JournalWriter categories = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("categories").$sym("category").index().buckets(100).$str("description").$ts())) {
            try (JournalWriter employees = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("employees").$str("employeeId").index().buckets(2048).$str("firstName").$str("lastName").$date("birthday").$ts())) {
                try (JournalWriter orderDetails = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("orderDetails").$int("orderDetailId").$int("orderId").$int("productId").$int("quantity").$ts())) {
                    try (JournalWriter orders = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("orders").$int("orderId").$int("customerId").index().$int("productId").$str("employeeId").index().$ts("orderDate").$sym("shipper").$())) {
                        try (JournalWriter products = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("products").$int("productId").$str("productName").$sym("supplier").index().buckets(100).$sym("category").index().buckets(100).$double("price").$ts())) {
                            try (JournalWriter shippers = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("shippers").$sym("shipper").$str("phone").$ts())) {
                                try (JournalWriter suppliers = FACTORY_CONTAINER.getFactory().writer(new JournalStructure("suppliers").$sym("supplier").buckets(100).$str("contactName").$str("address").$str("city").$str("postalCode").$sym("country").index().$str("phone").$ts())) {
                                    final Rnd rnd = new Rnd();
                                    long time = DateFormatUtils.parseDateTime("2015-07-10T00:00:00.000Z");
                                    // statics
                                    int countryCount = 196;
                                    ObjList<String> countries = new ObjList<>();
                                    for (int i = 0; i < countryCount; i++) {
                                        countries.add(rnd.nextString(rnd.nextInt() & 15));
                                    }
                                    IntHashSet blackList = new IntHashSet();
                                    // customers
                                    int customerCount = 10000;
                                    for (int i = 0; i < customerCount; i++) {
                                        if (rnd.nextPositiveInt() % 100 == 0) {
                                            blackList.add(i);
                                        }
                                        JournalEntryWriter w = customers.entryWriter();
                                        w.putInt(0, i);
                                        w.putStr(1, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(2, rnd.nextChars(rnd.nextInt() & 31));
                                        w.putStr(4, rnd.nextChars(rnd.nextInt() & 63));
                                        w.putStr(5, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putSym(6, countries.getQuick(rnd.nextPositiveInt() % 196));
                                        w.putDate(7, time++);
                                        w.append();
                                    }
                                    customers.commit();
                                    // categories
                                    for (int i = 0; i < 100; i++) {
                                        JournalEntryWriter w = categories.entryWriter();
                                        w.putSym(0, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(1, rnd.nextChars(rnd.nextInt() & 63));
                                        w.putDate(2, time++);
                                        w.append();
                                    }
                                    categories.commit();
                                    // employees
                                    int employeeCount = 2000;
                                    for (int i = 0; i < employeeCount; i++) {
                                        JournalEntryWriter w = employees.entryWriter();
                                        w.putStr(0, rnd.nextChars(rnd.nextInt() & 7));
                                        w.putStr(1, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(2, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putDate(3, 0);
                                        w.putDate(4, time++);
                                        w.append();
                                    }
                                    employees.commit();
                                    // suppliers
                                    for (int i = 0; i < 100; i++) {
                                        JournalEntryWriter w = suppliers.entryWriter();
                                        w.putSym(0, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(1, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(2, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(3, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(4, rnd.nextChars(rnd.nextInt() & 7));
                                        w.putSym(5, countries.getQuick(rnd.nextPositiveInt() % countryCount));
                                        w.putStr(6, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putDate(7, time++);
                                        w.append();
                                    }
                                    suppliers.commit();
                                    MMappedSymbolTable categoryTab = categories.getSymbolTable("category");
                                    int categoryTabSize = categoryTab.size();
                                    MMappedSymbolTable supplierTab = suppliers.getSymbolTable("supplier");
                                    int supplierTabSize = supplierTab.size();
                                    // products
                                    int productCount = 2000;
                                    for (int i = 0; i < productCount; i++) {
                                        JournalEntryWriter w = products.entryWriter();
                                        w.putInt(0, i);
                                        w.putStr(1, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putSym(2, supplierTab.value(rnd.nextPositiveInt() % supplierTabSize));
                                        w.putSym(3, categoryTab.value(rnd.nextPositiveInt() % categoryTabSize));
                                        w.putDouble(4, rnd.nextDouble());
                                        w.putDate(5, time++);
                                        w.append();
                                    }
                                    products.commit();
                                    // shippers
                                    for (int i = 0; i < 20; i++) {
                                        JournalEntryWriter w = shippers.entryWriter();
                                        w.putSym(0, rnd.nextChars(rnd.nextInt() & 15));
                                        w.putStr(1, rnd.nextChars(rnd.nextInt() & 7));
                                        w.append();
                                    }
                                    shippers.commit();
                                    MMappedSymbolTable shipperTab = shippers.getSymbolTable("shipper");
                                    int shipperTabSize = shipperTab.size();
                                    int d = 0;
                                    for (int i = 0; i < 100000; i++) {
                                        int customerId = rnd.nextPositiveInt() % customerCount;
                                        if (blackList.contains(customerId)) {
                                            continue;
                                        }
                                        int orderId = rnd.nextPositiveInt();
                                        JournalEntryWriter w = orders.entryWriter(time++);
                                        w.putInt(0, orderId);
                                        w.putInt(1, customerId);
                                        w.putInt(2, rnd.nextPositiveInt() % productCount);
                                        w.putStr(3, employees.getPartition(0, true).getFlyweightStr(rnd.nextPositiveLong() % employeeCount, 0));
                                        w.putSym(5, shipperTab.value(rnd.nextPositiveInt() % shipperTabSize));
                                        w.append();
                                        int k = (rnd.nextInt() & 3) + 1;
                                        for (int n = 0; n < k; n++) {
                                            JournalEntryWriter dw = orderDetails.entryWriter();
                                            dw.putInt(0, ++d);
                                            dw.putInt(1, orderId);
                                            dw.putInt(2, rnd.nextPositiveInt() % productCount);
                                            dw.putInt(3, (rnd.nextInt() & 3) + 1);
                                            dw.append();
                                        }
                                    }
                                    orders.commit();
                                    orderDetails.commit();
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}

Example 2 with JournalEntryWriter

use of com.questdb.store.JournalEntryWriter in project questdb by bluestreak01.

the class PlainTextStoringParser method onFields.

@Override
public void onFields(int line, ObjList<DirectByteCharSequence> values, int hi) {
    boolean append = true;
    try {
        JournalEntryWriter w = writer.entryWriter();
        for (int i = 0; i < hi; i++) {
            if (values.getQuick(i).length() == 0) {
                continue;
            }
            try {
                DirectByteCharSequence charField;
                ImportedColumnMetadata m = metadata.getQuick(i);
                switch(m.importedColumnType) {
                    case ColumnType.STRING:
                        utf8Sink.clear();
                        charField = values.getQuick(i);
                        Chars.utf8Decode(charField.getLo(), charField.getHi(), utf8Sink);
                        w.putStr(i, (DirectBytes) utf8Sink);
                        break;
                    case ColumnType.DOUBLE:
                        w.putDouble(i, Numbers.parseDouble(values.getQuick(i)));
                        break;
                    case ColumnType.INT:
                        w.putInt(i, Numbers.parseInt(values.getQuick(i)));
                        break;
                    case ColumnType.FLOAT:
                        w.putFloat(i, Numbers.parseFloat(values.getQuick(i)));
                        break;
                    case ColumnType.DATE:
                        if (m.dateFormat != null && m.dateLocale != null) {
                            w.putDate(i, m.dateFormat.parse(values.getQuick(i), m.dateLocale));
                        } else {
                            throw NumericException.INSTANCE;
                        }
                        break;
                    case ColumnType.SYMBOL:
                        utf8Sink.clear();
                        charField = values.getQuick(i);
                        Chars.utf8Decode(charField.getLo(), charField.getHi(), utf8Sink);
                        w.putSym(i, utf8Sink);
                        break;
                    case ColumnType.LONG:
                        w.putLong(i, Numbers.parseLong(values.getQuick(i)));
                        break;
                    case ColumnType.BOOLEAN:
                        w.putBool(i, Chars.equalsIgnoreCase(values.getQuick(i), "true"));
                        break;
                    default:
                        break;
                }
            } catch (Exception e) {
                switch(atomicity) {
                    case ATOMICITY_STRICT:
                        LOG.info().$("Error at (").$(line).$(',').$(i).$(')').$();
                        throw new JournalRuntimeException("Error on line: " + line + ", col: " + i);
                    default:
                        errors.increment(i);
                        LOG.debug().$("Error at (").$(line).$(',').$(i).$(") as ").$(metadata.getQuick(i).importedColumnType).$(": ").$(e.getMessage()).$();
                        append = false;
                }
                break;
            }
        }
        if (append) {
            w.append();
        }
    } catch (JournalException e) {
        throw new JournalRuntimeException(e);
    }
}

Example 3 with JournalEntryWriter

use of com.questdb.store.JournalEntryWriter in project questdb by bluestreak01.

the class QueryCompiler method copyNonPartitioned.

private void copyNonPartitioned(RecordCursor cursor, JournalWriter w, CopyHelper helper) throws JournalException {
    while (cursor.hasNext()) {
        Record r = cursor.next();
        JournalEntryWriter ew = w.entryWriter();
        helper.copy(r, ew);
        ew.append();
    }
}

Example 4 with JournalEntryWriter

use of com.questdb.store.JournalEntryWriter in project questdb by bluestreak01.

the class AppendObjectBlobs method processDir.

private static int processDir(JournalWriter writer, File dir) throws JournalException, IOException {
    int count = 0;
    File[] files = dir.listFiles();
    if (files != null) {
        for (File f : files) {
            if (f.isDirectory()) {
                count += processDir(writer, f);
                continue;
            }
            JournalEntryWriter w = writer.entryWriter(System.currentTimeMillis());
            w.putSym(0, f.getAbsolutePath());
            try (InputStream in = new FileInputStream(f)) {
                try (GZIPOutputStream out = new GZIPOutputStream(w.putBin(1))) {
                    pump(in, out);
                }
            }
            w.append();
            count++;
        }
        writer.commit();
    }
    return count;
}

Example 5 with JournalEntryWriter

use of com.questdb.store.JournalEntryWriter in project questdb by bluestreak01.

the class AppendRawTimeSeries method main.

public static void main(String[] args) throws JournalException, ParserException {
    if (args.length < 1) {
        System.out.println("Usage: AppendRawTimeSeries <path>");
        System.exit(1);
    }
    final String location = args[0];
    // factory can be reused in application and must be explicitly closed when no longer needed.
    try (Factory factory = new Factory(location, 1000, 1, 0)) {
        // to populate it.
        try (JournalWriter writer = factory.writer(new JournalStructure("customers").$int("id").$str("name").$ts("updateDate").$())) {
            Rnd rnd = new Rnd();
            long timestamp = System.currentTimeMillis();
            for (int i = 0; i < 1000000; i++) {
                // enforce timestamp order
                JournalEntryWriter ew = writer.entryWriter(timestamp);
                // columns accessed by index
                ew.putInt(0, rnd.nextPositiveInt());
                ew.putStr(1, rnd.nextChars(25));
                // increment timestamp by 30 seconds
                timestamp += 30000;
                // append record to journal
                ew.append();
            }
            // commit all records at once
            // there is no limit on how many records can be in the same transaction
            writer.commit();
        }
    }
}