Examples with EtlRun com.robertsanek.data.etl.EtlRun used on opensource projects

Example 1 with EtlRun

use of com.robertsanek.data.etl.EtlRun in project core by z1lc.

the class MasterEtl method runEtls.

@SuppressWarnings({ "rawtypes", "try" })
public boolean runEtls(boolean fastRun, boolean parallel) {
    Stopwatch total = Stopwatch.createStarted();
    List<Class<? extends Etl>> concreteEtls = getConcreteEtls(fastRun);
    log.info("Will run %s ETLs.", concreteEtls.size());
    log.info("Creating connection to Cloud SQL and re-generating table schemas... (this may take up to 3 minutes)");
    try (SessionFactory ignored = Unchecked.get(() -> getSessionFactory(Hbm2ddlType.CREATE, ConnectionType.RSANEK));
        SessionFactory noneSf = Unchecked.get(() -> getSessionFactory(Hbm2ddlType.NONE, ConnectionType.RSANEK))) {
        log.info("Schema re-generation complete, taking %s seconds. Beginning ETL with parallel = %s.", total.elapsed().getSeconds(), parallel);
        Stream<Class<? extends Etl>> stream = parallel ? concreteEtls.parallelStream() : concreteEtls.stream();
        List<EtlRun> etlRuns = stream.map(etlClazz -> runIndividualEtl(etlClazz, noneSf)).collect(Collectors.toList());
        try (Session session = noneSf.openSession()) {
            Transaction transaction = session.beginTransaction();
            etlRuns.forEach(session::save);
            session.flush();
            session.clear();
            transaction.commit();
        } catch (Exception e) {
            log.error(e);
        }
        log.info("Completed %s ETLs in %s seconds.", concreteEtls.size(), total.elapsed().getSeconds());
        return etlRuns.stream().allMatch(etlRun -> etlRun.getWas_successful() && etlRun.getRows_generated() > 0);
    }
}