Examples with TextFormatConfig org.apache.drill.exec.store.easy.text.TextFormatPlugin.TextFormatConfig used on opensource projects

Example 1 with TextFormatConfig

use of org.apache.drill.exec.store.easy.text.TextFormatPlugin.TextFormatConfig in project drill by apache.

the class TestCsv method setup.

@BeforeClass
public static void setup() throws Exception {
    startCluster(ClusterFixture.builder().maxParallelization(1));
    // Set up CSV storage plugin using headers.
    testDir = cluster.makeTempDir("csv");
    TextFormatConfig csvFormat = new TextFormatConfig();
    csvFormat.fieldDelimiter = ',';
    csvFormat.skipFirstLine = false;
    csvFormat.extractHeader = true;
    cluster.defineWorkspace("dfs", "data", testDir.getAbsolutePath(), "csv", csvFormat);
}

Example 2 with TextFormatConfig

use of org.apache.drill.exec.store.easy.text.TextFormatPlugin.TextFormatConfig in project drill by axbaretto.

the class TestCsv method setup.

@BeforeClass
public static void setup() throws Exception {
    startCluster(ClusterFixture.builder(dirTestWatcher).maxParallelization(1));
    // Set up CSV storage plugin using headers.
    TextFormatConfig csvFormat = new TextFormatConfig();
    csvFormat.fieldDelimiter = ',';
    csvFormat.skipFirstLine = false;
    csvFormat.extractHeader = true;
    testDir = cluster.makeDataDir("data", "csv", csvFormat);
}

Example 3 with TextFormatConfig

use of org.apache.drill.exec.store.easy.text.TextFormatPlugin.TextFormatConfig in project drill by apache.

the class TestRestJson method setup.

@BeforeClass
public static void setup() throws Exception {
    ClusterFixtureBuilder builder = new ClusterFixtureBuilder(dirTestWatcher).configProperty(ExecConstants.HTTP_ENABLE, true).configProperty(ExecConstants.HTTP_PORT_HUNT, true);
    startCluster(builder);
    portNumber = cluster.drillbit().getWebServerPort();
    // Set up CSV storage plugin using headers.
    TextFormatConfig csvFormat = new TextFormatConfig(null, // line delimiter
    null, // field delimiter
    null, // quote
    null, // escape
    null, // comment
    null, // skipFirstLine,
    false, // extractHeader
    true);
    testDir = cluster.makeDataDir("data", "csv", csvFormat);
}

Example 4 with TextFormatConfig

use of org.apache.drill.exec.store.easy.text.TextFormatPlugin.TextFormatConfig in project drill by apache.

the class DropboxFileSystemTest method setup.

/*
   All test files can be found in java-exec/src/test/resources/dropboxTestFiles

  Instructions for running Dropbox Unit Tests
  1.  Get your Dropbox API key as explained in the Drill docs and paste it above into the ACCESS_TOKEN variable.
  2.  In your dropbox account, create a folder called 'csv' and upload the file hdf-test.csvh into that folder
  3.  In your dropbox account, upload the file http-pcap.json to the root directory of your dropbox account
  4.  In the testListFiles test, you will have to update the modified dates
  5.  Run tests.
   */
@BeforeClass
public static void setup() throws Exception {
    assertTrue(!ACCESS_TOKEN.equalsIgnoreCase("<Your Dropbox Access Token Here>"));
    ClusterTest.startCluster(ClusterFixture.builder(dirTestWatcher));
    Map<String, String> dropboxConfigVars = new HashMap<>();
    dropboxConfigVars.put("dropboxAccessToken", ACCESS_TOKEN);
    // Create workspaces
    WorkspaceConfig rootWorkspace = new WorkspaceConfig("/", false, null, false);
    WorkspaceConfig csvWorkspace = new WorkspaceConfig("/csv", false, null, false);
    Map<String, WorkspaceConfig> workspaces = new HashMap<>();
    workspaces.put("root", rootWorkspace);
    workspaces.put("csv", csvWorkspace);
    // Add formats
    Map<String, FormatPluginConfig> formats = new HashMap<>();
    List<String> jsonExtensions = new ArrayList<>();
    jsonExtensions.add("json");
    FormatPluginConfig jsonFormatConfig = new JSONFormatConfig(jsonExtensions);
    // CSV Format
    List<String> csvExtensions = new ArrayList<>();
    csvExtensions.add("csv");
    csvExtensions.add("csvh");
    FormatPluginConfig csvFormatConfig = new TextFormatConfig(csvExtensions, "\n", ",", "\"", null, null, false, true);
    StoragePluginConfig dropboxConfig = new FileSystemConfig("dropbox:///", dropboxConfigVars, workspaces, formats, null);
    dropboxConfig.setEnabled(true);
    cluster.defineStoragePlugin("dropbox_test", dropboxConfig);
    cluster.defineFormat("dropbox_test", "json", jsonFormatConfig);
    cluster.defineFormat("dropbox_test", "csv", csvFormatConfig);
}

Example 5 with TextFormatConfig

use of org.apache.drill.exec.store.easy.text.TextFormatPlugin.TextFormatConfig in project drill by apache.

the class TestPluginRegistry method testFormatPlugin.

@Test
public void testFormatPlugin() throws Exception {
    ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher);
    try (ClusterFixture cluster = builder.build()) {
        StoragePluginRegistry registry = cluster.storageRegistry();
        StoragePluginConfig config = registry.getStoredConfig(CP_PLUGIN_NAME);
        FileSystemConfig fsConfig = (FileSystemConfig) config;
        assertFalse(fsConfig.getFormats().containsKey("bsv"));
        // Add a new format
        TextFormatConfig bsv = new TextFormatConfig(null, // line delimiter
        null, // field delimiter
        "!", // quote
        null, // escape
        null, // comment
        null, // skip first line
        false, // extract header
        false);
        registry.putFormatPlugin(CP_PLUGIN_NAME, "bsv", bsv);
        config = registry.getStoredConfig(CP_PLUGIN_NAME);
        fsConfig = (FileSystemConfig) config;
        assertTrue(fsConfig.getFormats().containsKey("bsv"));
        assertSame(bsv, fsConfig.getFormats().get("bsv"));
        // Remove the format
        registry.putFormatPlugin(CP_PLUGIN_NAME, "bsv", null);
        config = registry.getStoredConfig(CP_PLUGIN_NAME);
        fsConfig = (FileSystemConfig) config;
        assertFalse(fsConfig.getFormats().containsKey("bsv"));
        // Undefined plugin
        try {
            registry.putFormatPlugin("bogus", "bsv", bsv);
            fail();
        } catch (PluginException e) {
        // Expected
        }
        // Try to set a non-FS plugin
        try {
            registry.putFormatPlugin(SYS_PLUGIN_NAME, "bsv", bsv);
            fail();
        } catch (PluginException e) {
        // Expected
        }
    }
}