Examples with Pipe java.nio.channels.Pipe used on opensource projects

Example 11 with Pipe

use of java.nio.channels.Pipe in project beam by apache.

the class PackageUtilTest method testPackageUploadIsNotSkippedWhenSizesAreDifferent.

@Test
public void testPackageUploadIsNotSkippedWhenSizesAreDifferent() throws Exception {
    Pipe pipe = Pipe.open();
    File tmpDirectory = tmpFolder.newFolder("folder");
    tmpFolder.newFolder("folder", "empty_directory");
    tmpFolder.newFolder("folder", "directory");
    makeFileWithContents("folder/file.txt", "This is a test!");
    makeFileWithContents("folder/directory/file.txt", "This is also a test!");
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(createStorageObject(STAGING_PATH, Long.MAX_VALUE))));
    when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
    PackageUtil.stageClasspathElements(ImmutableList.of(tmpDirectory.getAbsolutePath()), STAGING_PATH, createOptions);
    verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
    verify(mockGcsUtil).create(any(GcsPath.class), anyString());
    verifyNoMoreInteractions(mockGcsUtil);
}

Example 12 with Pipe

use of java.nio.channels.Pipe in project beam by apache.

the class PackageUtilTest method testPackageUploadEventuallySucceeds.

@Test
public void testPackageUploadEventuallySucceeds() throws Exception {
    Pipe pipe = Pipe.open();
    File tmpFile = makeFileWithContents("file.txt", "This is a test!");
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException("some/path"))));
    when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenThrow(// First attempt fails
    new IOException("Fake Exception: 410 Gone")).thenReturn(// second attempt succeeds
    pipe.sink());
    try {
        PackageUtil.stageClasspathElements(ImmutableList.of(tmpFile.getAbsolutePath()), STAGING_PATH, fastNanoClockAndSleeper, MoreExecutors.newDirectExecutorService(), createOptions);
    } finally {
        verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
        verify(mockGcsUtil, times(2)).create(any(GcsPath.class), anyString());
        verifyNoMoreInteractions(mockGcsUtil);
    }
}

Example 13 with Pipe

use of java.nio.channels.Pipe in project beam by apache.

the class PackageUtilTest method testPackageUploadWithFileSucceeds.

@Test
public void testPackageUploadWithFileSucceeds() throws Exception {
    Pipe pipe = Pipe.open();
    String contents = "This is a test!";
    File tmpFile = makeFileWithContents("file.txt", contents);
    when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))).thenReturn(ImmutableList.of(StorageObjectOrIOException.create(new FileNotFoundException("some/path"))));
    when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
    List<DataflowPackage> targets = PackageUtil.stageClasspathElements(ImmutableList.of(tmpFile.getAbsolutePath()), STAGING_PATH, createOptions);
    DataflowPackage target = Iterables.getOnlyElement(targets);
    verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class));
    verify(mockGcsUtil).create(any(GcsPath.class), anyString());
    verifyNoMoreInteractions(mockGcsUtil);
    assertThat(target.getName(), RegexMatcher.matches("file-" + HASH_PATTERN + ".txt"));
    assertThat(target.getLocation(), equalTo(STAGING_PATH + target.getName()));
    assertThat(new LineReader(Channels.newReader(pipe.source(), "UTF-8")).readLine(), equalTo(contents));
}

Example 14 with Pipe

use of java.nio.channels.Pipe in project j2objc by google.

the class PipeTest method test_readEmptyPipe.

public void test_readEmptyPipe() throws Exception {
    Pipe p = Pipe.open();
    p.source().configureBlocking(false);
    assertEquals(0, p.source().read(ByteBuffer.allocate(1)));
}

Example 15 with Pipe

use of java.nio.channels.Pipe in project databus by linkedin.

the class TestGenericDispatcher method testShutdownBeforeRollback.

@Test(groups = { "small", "functional" })
public /**
     *
     * 1. Dispatcher is dispatching 2 window of events.
     * 2. First window consumption is successfully done.
     * 3. The second window's onStartDataEventSequence() of the callback registered is blocked (interruptible),
     *    causing the dispatcher to wait.
     * 4. At this instant the dispatcher is shut down. The callback is made to return Failure status which would
     *    cause rollback in normal scenario.
     * 5. As the shutdown message is passed, the blocked callback is expected to be interrupted and no rollback
     *    calls MUST be made.
     */
void testShutdownBeforeRollback() throws Exception {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testShutdownBeforeRollback");
    log.setLevel(Level.INFO);
    //log.getRoot().setLevel(Level.DEBUG);
    LOG.info("start");
    // generate events
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add((short) 1);
    DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    final int numEvents = 8;
    final int numEventsPerWindow = 4;
    final int payloadSize = 200;
    Assert.assertTrue(evGen.generateEvents(numEvents, numEventsPerWindow, 500, payloadSize, srcTestEvents) > 0);
    // find out how much data we need to stream for the failure
    // account for the EOW event which is < payload size
    int win1Size = payloadSize - 1;
    for (DbusEvent e : srcTestEvents) {
        win1Size += e.size();
    }
    //serialize the events to a buffer so they can be sent to the client
    final TestGenericDispatcherEventBuffer srcEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, srcEventsBuf, null, true);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    //Create destination (client) buffer
    final TestGenericDispatcherEventBuffer destEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    /**
       *
       *  Consumer with ability to wait for latch during onStartDataEventSequence()
       */
    class TimeoutDESConsumer extends TimeoutTestConsumer {

        private final CountDownLatch latch = new CountDownLatch(1);

        private int _countStartWindow = 0;

        private final int _failedRequestNumber;

        public TimeoutDESConsumer(int failedRequestNumber) {
            super(1, 1, 0, 0, 0, 0);
            _failedRequestNumber = failedRequestNumber;
        }

        public CountDownLatch getLatch() {
            return latch;
        }

        @Override
        public ConsumerCallbackResult onStartDataEventSequence(SCN startScn) {
            _countStartWindow++;
            if (_countStartWindow == _failedRequestNumber) {
                // Wait for the latch to open
                try {
                    latch.await();
                } catch (InterruptedException e) {
                }
                return ConsumerCallbackResult.ERROR_FATAL;
            }
            return super.onStartDataEventSequence(startScn);
        }

        @Override
        public ConsumerCallbackResult onDataEvent(DbusEvent e, DbusEventDecoder eventDecoder) {
            return ConsumerCallbackResult.SUCCESS;
        }

        public int getNumBeginWindowCalls() {
            return _countStartWindow;
        }
    }
    //Create dispatcher
    //fail on second window
    final TimeoutDESConsumer mockConsumer = new TimeoutDESConsumer(2);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer((DatabusStreamConsumer) mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    final ConsumerCallbackStats callbackStats = new ConsumerCallbackStats(0, "test", "test", true, false, null);
    final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), // 100 ms budget
    100, new StreamConsumerCallbackFactory(callbackStats, unifiedStats), callbackStats, unifiedStats, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), destEventsBuf, callback, null, null, null, null, null);
    final Thread dispatcherThread = new Thread(dispatcher);
    log.info("starting dispatcher thread");
    dispatcherThread.start();
    // Generate RegisterRespone for schema
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    // Enqueue Necessary messages before starting dispatch
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    log.info("starting event dispatch");
    //comm channels between reader and writer
    Pipe pipe = Pipe.open();
    Pipe.SinkChannel writerStream = pipe.sink();
    Pipe.SourceChannel readerStream = pipe.source();
    writerStream.configureBlocking(true);
    /*
       *  Needed for DbusEventBuffer.readEvents() to exit their loops when no more data is available.
       *  With Pipe mimicking ChunkedBodyReadableByteChannel, we need to make Pipe non-blocking on the
       *  reader side to achieve the behavior that ChunkedBodyReadableByte channel provides.
       */
    readerStream.configureBlocking(false);
    //Event writer - Relay in the real world
    Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
    //Event readers - Clients in the real world
    //Checkpoint pullerCheckpoint = Checkpoint.createFlexibleCheckpoint();
    DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(0, "client", true, false, null);
    DbusEventBufferReader reader = new DbusEventBufferReader(destEventsBuf, readerStream, null, clientStats);
    UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader, "Reader");
    tReader.setDaemon(true);
    tReader.start();
    try {
        log.info("send both windows");
        StreamEventsResult streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win1Size));
        // EOP events, presumably?
        Assert.assertEquals(// EOP events, presumably?
        "num events streamed should equal total number of events plus 2", numEvents + 2, streamRes.getNumEventsStreamed());
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return 2 == mockConsumer.getNumBeginWindowCalls();
            }
        }, "second window processing started", 5000, log);
        dispatcher.shutdown();
        // remove the barrier
        mockConsumer.getLatch().countDown();
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return !dispatcherThread.isAlive();
            }
        }, "Ensure Dispatcher thread is shutdown", 5000, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return 0 == mockConsumer.getNumRollbacks();
            }
        }, "Ensure No Rollback is called", 10, log);
    } finally {
        reader.stop();
    }
    log.info("end\n");
}