Examples with FileSystemViewStorageConfig org.apache.hudi.common.table.view.FileSystemViewStorageConfig used on opensource projects

Example 1 with FileSystemViewStorageConfig

use of org.apache.hudi.common.table.view.FileSystemViewStorageConfig in project hudi by apache.

the class TestRemoteHoodieTableFileSystemView method getFileSystemView.

protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) {
    FileSystemViewStorageConfig sConf = FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.SPILLABLE_DISK).build();
    HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().build();
    HoodieCommonConfig commonConfig = HoodieCommonConfig.newBuilder().build();
    HoodieLocalEngineContext localEngineContext = new HoodieLocalEngineContext(metaClient.getHadoopConf());
    try {
        server = new TimelineService(localEngineContext, new Configuration(), TimelineService.Config.builder().serverPort(0).build(), FileSystem.get(new Configuration()), FileSystemViewManager.createViewManager(localEngineContext, metadataConfig, sConf, commonConfig));
        server.startService();
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
    LOG.info("Connecting to Timeline Server :" + server.getServerPort());
    view = new RemoteHoodieTableFileSystemView("localhost", server.getServerPort(), metaClient);
    return view;
}

Example 2 with FileSystemViewStorageConfig

use of org.apache.hudi.common.table.view.FileSystemViewStorageConfig in project hudi by apache.

the class StreamerUtil method getHoodieClientConfig.

public static HoodieWriteConfig getHoodieClientConfig(Configuration conf, boolean enableEmbeddedTimelineService, boolean loadFsViewStorageConfig) {
    HoodieWriteConfig.Builder builder = HoodieWriteConfig.newBuilder().withEngineType(EngineType.FLINK).withPath(conf.getString(FlinkOptions.PATH)).combineInput(conf.getBoolean(FlinkOptions.PRE_COMBINE), true).withMergeAllowDuplicateOnInserts(OptionsResolver.insertClustering(conf)).withCompactionConfig(HoodieCompactionConfig.newBuilder().withPayloadClass(conf.getString(FlinkOptions.PAYLOAD_CLASS_NAME)).withTargetIOPerCompactionInMB(conf.getLong(FlinkOptions.COMPACTION_TARGET_IO)).withInlineCompactionTriggerStrategy(CompactionTriggerStrategy.valueOf(conf.getString(FlinkOptions.COMPACTION_TRIGGER_STRATEGY).toUpperCase(Locale.ROOT))).withMaxNumDeltaCommitsBeforeCompaction(conf.getInteger(FlinkOptions.COMPACTION_DELTA_COMMITS)).withMaxDeltaSecondsBeforeCompaction(conf.getInteger(FlinkOptions.COMPACTION_DELTA_SECONDS)).withAsyncClean(conf.getBoolean(FlinkOptions.CLEAN_ASYNC_ENABLED)).retainCommits(conf.getInteger(FlinkOptions.CLEAN_RETAIN_COMMITS)).withCleanerParallelism(20).archiveCommitsWith(conf.getInteger(FlinkOptions.ARCHIVE_MIN_COMMITS), conf.getInteger(FlinkOptions.ARCHIVE_MAX_COMMITS)).withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS).build()).withMemoryConfig(HoodieMemoryConfig.newBuilder().withMaxMemoryMaxSize(conf.getInteger(FlinkOptions.WRITE_MERGE_MAX_MEMORY) * 1024 * 1024L, conf.getInteger(FlinkOptions.COMPACTION_MAX_MEMORY) * 1024 * 1024L).build()).forTable(conf.getString(FlinkOptions.TABLE_NAME)).withStorageConfig(HoodieStorageConfig.newBuilder().logFileDataBlockMaxSize(conf.getInteger(FlinkOptions.WRITE_LOG_BLOCK_SIZE) * 1024 * 1024).logFileMaxSize(conf.getLong(FlinkOptions.WRITE_LOG_MAX_SIZE) * 1024 * 1024).parquetBlockSize(conf.getInteger(FlinkOptions.WRITE_PARQUET_BLOCK_SIZE) * 1024 * 1024).parquetPageSize(conf.getInteger(FlinkOptions.WRITE_PARQUET_PAGE_SIZE) * 1024 * 1024).parquetMaxFileSize(conf.getInteger(FlinkOptions.WRITE_PARQUET_MAX_FILE_SIZE) * 1024 * 1024L).build()).withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(conf.getBoolean(FlinkOptions.METADATA_ENABLED)).withMaxNumDeltaCommitsBeforeCompaction(conf.getInteger(FlinkOptions.METADATA_COMPACTION_DELTA_COMMITS)).build()).withPayloadConfig(HoodiePayloadConfig.newBuilder().withPayloadOrderingField(conf.getString(FlinkOptions.PRECOMBINE_FIELD)).withPayloadEventTimeField(conf.getString(FlinkOptions.PRECOMBINE_FIELD)).build()).withEmbeddedTimelineServerEnabled(enableEmbeddedTimelineService).withEmbeddedTimelineServerReuseEnabled(// make write client embedded timeline service singleton
    true).withAutoCommit(false).withAllowOperationMetadataField(conf.getBoolean(FlinkOptions.CHANGELOG_ENABLED)).withProps(flinkConf2TypedProperties(conf)).withSchema(getSourceSchema(conf).toString());
    HoodieWriteConfig writeConfig = builder.build();
    if (loadFsViewStorageConfig) {
        // do not use the builder to give a change for recovering the original fs view storage config
        FileSystemViewStorageConfig viewStorageConfig = ViewStorageProperties.loadFromProperties(conf.getString(FlinkOptions.PATH));
        writeConfig.setViewStorageConfig(viewStorageConfig);
    }
    return writeConfig;
}

Example 3 with FileSystemViewStorageConfig

use of org.apache.hudi.common.table.view.FileSystemViewStorageConfig in project hudi by apache.

the class TestTimelineServerBasedWriteMarkers method setup.

@BeforeEach
public void setup() throws IOException {
    initPath();
    initMetaClient();
    this.jsc = new JavaSparkContext(HoodieClientTestUtils.getSparkConfForTest(TestTimelineServerBasedWriteMarkers.class.getName()));
    this.context = new HoodieSparkEngineContext(jsc);
    this.fs = FSUtils.getFs(metaClient.getBasePath(), metaClient.getHadoopConf());
    this.markerFolderPath = new Path(metaClient.getMarkerFolderPath("000"));
    FileSystemViewStorageConfig storageConf = FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.SPILLABLE_DISK).build();
    HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().build();
    HoodieLocalEngineContext localEngineContext = new HoodieLocalEngineContext(metaClient.getHadoopConf());
    try {
        timelineService = new TimelineService(localEngineContext, new Configuration(), TimelineService.Config.builder().serverPort(0).enableMarkerRequests(true).build(), FileSystem.get(new Configuration()), FileSystemViewManager.createViewManager(localEngineContext, metadataConfig, storageConf, HoodieCommonConfig.newBuilder().build()));
        timelineService.startService();
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
    this.writeMarkers = new TimelineServerBasedWriteMarkers(metaClient.getBasePath(), markerFolderPath.toString(), "000", "localhost", timelineService.getServerPort(), 300);
}

Example 4 with FileSystemViewStorageConfig

use of org.apache.hudi.common.table.view.FileSystemViewStorageConfig in project hudi by apache.

the class StreamerUtil method createWriteClient.

/**
 * Creates the Flink write client.
 *
 * <p>This expects to be used by the driver, the client can then send requests for files view.
 *
 * <p>The task context supplier is a constant: the write token is always '0-1-0'.
 */
@SuppressWarnings("rawtypes")
public static HoodieFlinkWriteClient createWriteClient(Configuration conf) throws IOException {
    HoodieWriteConfig writeConfig = getHoodieClientConfig(conf, true, false);
    // build the write client to start the embedded timeline server
    final HoodieFlinkWriteClient writeClient = new HoodieFlinkWriteClient<>(HoodieFlinkEngineContext.DEFAULT, writeConfig);
    // create the filesystem view storage properties for client
    final FileSystemViewStorageConfig viewStorageConfig = writeConfig.getViewStorageConfig();
    // rebuild the view storage config with simplified options.
    FileSystemViewStorageConfig rebuilt = FileSystemViewStorageConfig.newBuilder().withStorageType(viewStorageConfig.getStorageType()).withRemoteServerHost(viewStorageConfig.getRemoteViewServerHost()).withRemoteServerPort(viewStorageConfig.getRemoteViewServerPort()).withRemoteTimelineClientTimeoutSecs(viewStorageConfig.getRemoteTimelineClientTimeoutSecs()).build();
    ViewStorageProperties.createProperties(conf.getString(FlinkOptions.PATH), rebuilt);
    return writeClient;
}

Example 5 with FileSystemViewStorageConfig

use of org.apache.hudi.common.table.view.FileSystemViewStorageConfig in project hudi by apache.

the class TestWriteCopyOnWrite method testReuseEmbeddedServer.

@Test
public void testReuseEmbeddedServer() throws IOException {
    conf.setInteger("hoodie.filesystem.view.remote.timeout.secs", 500);
    HoodieFlinkWriteClient writeClient = StreamerUtil.createWriteClient(conf);
    FileSystemViewStorageConfig viewStorageConfig = writeClient.getConfig().getViewStorageConfig();
    assertSame(viewStorageConfig.getStorageType(), FileSystemViewStorageType.REMOTE_FIRST);
    // get another write client
    writeClient = StreamerUtil.createWriteClient(conf);
    assertSame(writeClient.getConfig().getViewStorageConfig().getStorageType(), FileSystemViewStorageType.REMOTE_FIRST);
    assertEquals(viewStorageConfig.getRemoteViewServerPort(), writeClient.getConfig().getViewStorageConfig().getRemoteViewServerPort());
    assertEquals(viewStorageConfig.getRemoteTimelineClientTimeoutSecs(), 500);
}