Examples with Importer com.cinchapi.concourse.importer.Importer used on opensource projects

Example 1 with Importer

use of com.cinchapi.concourse.importer.Importer in project concourse by cinchapi.

the class ExportCliTest method beforeEachTest.

@Override
public void beforeEachTest() {
    String file = Resources.getAbsolutePath("/college.csv");
    Importer importer = new CsvImporter(client);
    importer.importFile(file);
    output = FileOps.tempFile();
}

Example 2 with Importer

use of com.cinchapi.concourse.importer.Importer in project concourse by cinchapi.

the class FindCriteriaTest method testSimpleWithTime.

@Test
public void testSimpleWithTime() {
    Set<Long> results = client.find(Criteria.where().key("graduation_rate").operator(Operator.GREATER_THAN).value(90));
    Timestamp t1 = Timestamp.now();
    System.out.println("Importing college data into Concourse");
    Importer importer = new CsvImporter(client);
    importer.importFile(Resources.get("/college.csv").getFile());
    Assert.assertEquals(results, client.find(Criteria.where().key("graduation_rate").operator(Operator.GREATER_THAN).value(90).at(t1)));
}

Example 3 with Importer

use of com.cinchapi.concourse.importer.Importer in project concourse by cinchapi.

the class FindCriteriaTest method beforeEachTest.

@Override
protected void beforeEachTest() {
    // Import data into Concourse
    System.out.println("Importing college data into Concourse");
    Importer importer = new CsvImporter(client);
    importer.importFile(Resources.get("/college.csv").getFile());
    // Load up the SQL db which also contains a copy of the data
    System.out.println("Loading SQL database with college data");
    try {
        // NOTE: The JDBC API is atrocious :o=
        Class.forName("org.sqlite.JDBC");
        Connection conn = DriverManager.getConnection("jdbc:sqlite:" + Resources.get("/college.db").getFile());
        sql = conn.createStatement();
    } catch (Exception e) {
        throw CheckedExceptions.wrapAsRuntimeException(e);
    }
    super.beforeEachTest();
}

Example 4 with Importer

use of com.cinchapi.concourse.importer.Importer in project concourse by cinchapi.

the class ImportCli method doTask.

@Override
protected void doTask() {
    final ImportOptions opts = (ImportOptions) options;
    final Set<Long> records;
    final Constructor<? extends Importer> constructor = getConstructor(opts.type);
    this.dryRun = opts.dryRun;
    opts.dynamic.put(Importer.ANNOTATE_DATA_SOURCE_OPTION_NAME, Boolean.toString(opts.annotateDataSource));
    if (opts.data == null) {
        // Import data from stdin
        Importer importer = Reflection.newInstance(constructor, getConcourseConnection(0));
        if (!opts.dynamic.isEmpty()) {
            importer.setParams(options.dynamic);
        }
        if (importer instanceof Headered && !opts.header.isEmpty()) {
            ((Headered) importer).parseHeader(opts.header);
        }
        try {
            ConsoleReader reader = new ConsoleReader();
            String line;
            records = Sets.newLinkedHashSet();
            Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {

                // Interactive import is ended when user presses CTRL + C,
                // so we need this shutdown hook to ensure that they get
                // feedback about the import before the JVM dies.
                @Override
                public void run() {
                    if (options.verbose) {
                        System.out.println(records);
                    }
                    System.out.println(AnyStrings.format("Imported data into {} record{}", records.size(), records.size() > 1 ? "s" : ""));
                    if (dryRun) {
                        System.out.println(((ImportDryRunConcourse) getConcourseConnection(0)).dump());
                    }
                }
            }));
            try {
                final AtomicBoolean lock = new AtomicBoolean(false);
                new Thread(new // If there is no input in
                Runnable() {

                    // 100ms, assume that the
                    // session is interactive (i.e.
                    // not piped) and display a
                    // prompt
                    @Override
                    public void run() {
                        try {
                            Thread.sleep(100);
                            if (lock.compareAndSet(false, true)) {
                                System.out.println("Importing from stdin. Press " + "CTRL + C when finished");
                            }
                        } catch (InterruptedException e) {
                        }
                    }
                }).start();
                while ((line = reader.readLine()) != null) {
                    try {
                        lock.set(true);
                        records.addAll(importer.importString(line));
                    } catch (Exception e) {
                        System.err.println(e);
                    }
                }
            } catch (IOException e) {
                throw Throwables.propagate(e);
            }
        } catch (IOException e) {
            throw Throwables.propagate(e);
        } finally {
            try {
                TerminalFactory.get().restore();
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }
    } else {
        String path = FileOps.expandPath(opts.data, getLaunchDirectory());
        Collection<String> files = FileOps.isDirectory(path) ? scan(Paths.get(path)) : ImmutableList.of(path);
        Stopwatch watch = Stopwatch.createUnstarted();
        if (files.size() > 1) {
            records = Sets.newConcurrentHashSet();
            final Queue<String> filesQueue = (Queue<String>) files;
            List<Runnable> runnables = Lists.newArrayListWithCapacity(opts.numThreads);
            // Create just enough Runnables with instantiated Importers in
            // advance. Each of those Runnables will work until #filesQueue
            // is exhausted.
            opts.numThreads = Math.min(opts.numThreads, files.size());
            for (int i = 0; i < opts.numThreads; ++i) {
                final Importer importer0 = Reflection.newInstance(constructor, getConcourseConnection(i));
                if (!opts.dynamic.isEmpty()) {
                    importer0.setParams(opts.dynamic);
                }
                if (importer0 instanceof Headered && !opts.header.isEmpty()) {
                    ((Headered) importer0).parseHeader(opts.header);
                }
                runnables.add(new Runnable() {

                    private final Importer importer = importer0;

                    @Override
                    public void run() {
                        String file;
                        while ((file = filesQueue.poll()) != null) {
                            records.addAll(importer.importFile(file));
                        }
                    }
                });
            }
            ExecutorService executor = Executors.newFixedThreadPool(runnables.size());
            System.out.println("Starting import...");
            watch.start();
            for (Runnable runnable : runnables) {
                executor.execute(runnable);
            }
            executor.shutdown();
            try {
                if (!executor.awaitTermination(1, TimeUnit.MINUTES)) {
                    while (!executor.isTerminated()) {
                        // block until all tasks are
                        System.out.print('.');
                    // completed and provide some
                    // feedback to the user
                    }
                }
            } catch (InterruptedException e) {
                throw Throwables.propagate(e);
            }
        } else {
            Importer importer = Reflection.newInstance(constructor, getConcourseConnection(0));
            if (!opts.dynamic.isEmpty()) {
                importer.setParams(opts.dynamic);
            }
            if (importer instanceof Headered && !opts.header.isEmpty()) {
                ((Headered) importer).parseHeader(opts.header);
            }
            System.out.println("Starting import...");
            watch.start();
            records = importer.importFile(files.iterator().next());
        }
        watch.stop();
        long elapsed = watch.elapsed(TimeUnit.MILLISECONDS);
        double seconds = elapsed / 1000.0;
        if (options.verbose) {
            System.out.println(records);
        }
        System.out.println(AnyStrings.format("Imported data into {} record{} in {} seconds", records.size(), records.size() > 1 ? "s" : "", seconds));
        if (dryRun) {
            System.out.println(((ImportDryRunConcourse) getConcourseConnection(0)).dump());
        }
    }
}

Example 5 with Importer

use of com.cinchapi.concourse.importer.Importer in project concourse by cinchapi.

the class ImportCli method getCustomImporterClass.

/**
 * Given an alias (or fully qualified class name) attempt to load a "custom"
 * importer that is not already defined in the {@link #importers built-in}
 * collection.
 *
 * @param alias a conventional alias (i.e. FileTypeImporter --> file-type)
 *            OR a fully qualified class name OR the path to a customer
 *            importer file (can be .groovy or .jar) OR the name of a
 *            customer importer file contained in the "importers" directory
 *            of the server's home
 * @return the {@link Class} that corresponds to the custom importer
 * @throws ClassNotFoundException
 */
@SuppressWarnings("unchecked")
private static Class<? extends Importer> getCustomImporterClass(String alias) throws ClassNotFoundException {
    try {
        return (Class<? extends Importer>) Class.forName(alias);
    } catch (ClassNotFoundException e) {
        Path path;
        boolean exists = true;
        if (!(path = Paths.get(Files.expandPath(alias, LAUNCH_DIRECTORY))).toFile().exists()) {
            exists = false;
            Path concourseServerHome = getConcourseServerHome();
            if (concourseServerHome != null) {
                Path importers = getConcourseServerHome().resolve("importers");
                String[] candidates = { alias, alias + ".groovy", alias + ".jar" };
                for (String candidate : candidates) {
                    if ((path = importers.resolve(candidate)).toFile().exists()) {
                        exists = true;
                        break;
                    }
                }
            }
        }
        if (exists) {
            if (path.toString().endsWith(".groovy")) {
                return GroovyFiles.loadClass(path);
            } else if (path.toString().endsWith(".jar")) {
                URLClassLoader typeClassLoader = null;
                URLClassLoader serverClassLoader = null;
                try {
                    URL[] urls = new URL[] { path.toFile().toURI().toURL() };
                    typeClassLoader = new URLClassLoader(urls, null);
                    serverClassLoader = new URLClassLoader(urls, Thread.currentThread().getContextClassLoader());
                    ClassPath classpath = ClassPath.from(typeClassLoader);
                    for (ClassInfo info : classpath.getAllClasses()) {
                        String name = info.getName();
                        Class<?> clazz = serverClassLoader.loadClass(name);
                        if (Importer.class.isAssignableFrom((clazz))) {
                            return (Class<? extends Importer>) clazz;
                        }
                    }
                    throw e;
                } catch (IOException ioe) {
                    throw CheckedExceptions.throwAsRuntimeException(ioe);
                } finally {
                    for (URLClassLoader loader : ImmutableList.of(typeClassLoader, serverClassLoader)) {
                        if (loader != null) {
                            try {
                                loader.close();
                            } catch (IOException ignore) {
                                throw CheckedExceptions.throwAsRuntimeException(ignore);
                            }
                        }
                    }
                }
            } else {
                throw new UnsupportedOperationException(AnyStrings.format("{} is an unsupported file type for custom importers", path));
            }
        } else {
            // Attempt to determine the correct class name from the alias by
            // loading the server's classpath. For the record, this is hella
            // slow.
            // turn off reflection logging
            Reflections.log = null;
            Reflections reflections = new Reflections();
            char firstChar = alias.charAt(0);
            for (Class<? extends Importer> clazz : reflections.getSubTypesOf(Importer.class)) {
                String name = clazz.getSimpleName();
                if (name.length() == 0) {
                    // Skip anonymous subclasses
                    continue;
                }
                char nameFirstChar = name.charAt(0);
                if (!Modifier.isAbstract(clazz.getModifiers()) && (nameFirstChar == Character.toUpperCase(firstChar) || nameFirstChar == Character.toLowerCase(firstChar))) {
                    String expected = CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_HYPHEN, clazz.getSimpleName()).replaceAll("-importer", "");
                    if (alias.equals(expected)) {
                        return clazz;
                    }
                }
            }
        }
        throw e;
    }
}