Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for vr goggle phone holders (Split display rendering of video) #92

Open
wants to merge 3 commits into
base: dev
Choose a base branch
from
Open

Add support for vr goggle phone holders (Split display rendering of video) #92

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
10ad6d8
Add support for vr goggle phone holders (Split display rendering of v…
Jun 18, 2021
1efd112
Refactor code into a new custom view "VrView". Move all gles helper …
Jul 17, 2021
c95962f
Fix bug where video doesn't start right away and fix possible teardow…
Jul 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion app/src/main/java/com/fpvout/digiview/MainActivity.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import androidx.appcompat.app.AppCompatActivity;
import androidx.preference.PreferenceManager;

import com.fpvout.digiview.vr.views.VrView;

import java.util.HashMap;

import io.sentry.SentryLevel;
Expand Down Expand Up @@ -113,7 +115,9 @@ public void onClick(View v) {
mUsbMaskConnection = new UsbMaskConnection();
Handler videoReaderEventListener = new Handler(this.getMainLooper(), msg -> onVideoReaderEvent((VideoReaderExoplayer.VideoReaderEventMessageCode) msg.obj));

mVideoReader = new VideoReaderExoplayer(fpvView, this, videoReaderEventListener);
VrView vrView = findViewById(R.id.vr_view);

mVideoReader = new VideoReaderExoplayer(fpvView, vrView, this, videoReaderEventListener);

if (!usbConnected) {
if (searchDevice()) {
Expand Down
178 changes: 105 additions & 73 deletions app/src/main/java/com/fpvout/digiview/VideoReaderExoplayer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,27 @@

import android.content.Context;
import android.content.SharedPreferences;
import android.graphics.SurfaceTexture;
import android.net.Uri;
import android.opengl.GLES20;
import android.os.Handler;
import android.os.Looper;
import android.os.Message;
import android.util.Log;
import android.view.Surface;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import android.view.View;

import androidx.annotation.NonNull;
import androidx.constraintlayout.widget.ConstraintLayout;
import androidx.preference.PreferenceManager;

import com.fpvout.digiview.vr.gles.EglCore;
import com.fpvout.digiview.vr.gles.FullFrameRect;
import com.fpvout.digiview.vr.gles.Texture2dProgram;
import com.fpvout.digiview.vr.gles.WindowSurface;
import com.fpvout.digiview.vr.views.VrView;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.DefaultLoadControl;
import com.google.android.exoplayer2.ExoPlaybackException;
Expand All @@ -37,22 +48,30 @@ public class VideoReaderExoplayer {
private SimpleExoPlayer mPlayer;
static final String VideoPreset = "VideoPreset";
private final SurfaceView surfaceView;
private final VrView vrView;
private AndroidUSBInputStream inputStream;
private UsbMaskConnection mUsbMaskConnection;
private UsbMaskConnection mUsbMaskConnection;
private boolean zoomedIn;
private final Context context;
private PerformancePreset performancePreset = PerformancePreset.getPreset(PerformancePreset.PresetType.DEFAULT);
static final String VideoZoomedIn = "VideoZoomedIn";
static final String VREnabled = "VREnabled";
private final SharedPreferences sharedPreferences;

VideoReaderExoplayer(SurfaceView videoSurface, Context c) {
VideoReaderExoplayer(SurfaceView videoSurface,
VrView vrView,
Context c) {
surfaceView = videoSurface;
this.vrView = vrView;
context = c;
sharedPreferences = PreferenceManager.getDefaultSharedPreferences(c);
}

VideoReaderExoplayer(SurfaceView videoSurface, Context c, Handler v) {
this(videoSurface, c);
VideoReaderExoplayer(SurfaceView videoSurface,
VrView vrView,
Context c,
Handler v) {
this(videoSurface, vrView, c);
videoReaderEventListener = v;
}

Expand All @@ -63,87 +82,97 @@ public void setUsbMaskConnection(UsbMaskConnection connection) {

public void start() {
zoomedIn = sharedPreferences.getBoolean(VideoZoomedIn, true);
boolean vrEnabled = sharedPreferences.getBoolean(VREnabled, false);
performancePreset = PerformancePreset.getPreset(sharedPreferences.getString(VideoPreset, "default"));

DefaultLoadControl loadControl = new DefaultLoadControl.Builder().setBufferDurationsMs(performancePreset.exoPlayerMinBufferMs, performancePreset.exoPlayerMaxBufferMs, performancePreset.exoPlayerBufferForPlaybackMs, performancePreset.exoPlayerBufferForPlaybackAfterRebufferMs).build();
mPlayer = new SimpleExoPlayer.Builder(context).setLoadControl(loadControl).build();
DefaultLoadControl loadControl = new DefaultLoadControl.Builder().setBufferDurationsMs(performancePreset.exoPlayerMinBufferMs, performancePreset.exoPlayerMaxBufferMs, performancePreset.exoPlayerBufferForPlaybackMs, performancePreset.exoPlayerBufferForPlaybackAfterRebufferMs).build();
mPlayer = new SimpleExoPlayer.Builder(context).setLoadControl(loadControl).build();
if (vrEnabled) {
vrView.setVisibility(View.VISIBLE);
vrView.start(mPlayer);
} else {
vrView.stop();
vrView.setVisibility(View.GONE);
mPlayer.setVideoSurfaceView(surfaceView);
mPlayer.setVideoScalingMode(C.VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING);
mPlayer.setWakeMode(C.WAKE_MODE_LOCAL);
}

DataSpec dataSpec = new DataSpec(Uri.EMPTY, 0, C.LENGTH_UNSET);
mPlayer.setVideoScalingMode(C.VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING);
mPlayer.setWakeMode(C.WAKE_MODE_LOCAL);

Log.d(TAG, "preset: " + performancePreset);
DataSpec dataSpec = new DataSpec(Uri.EMPTY, 0, C.LENGTH_UNSET);

DataSource.Factory dataSourceFactory = () -> {
switch (performancePreset.dataSourceType){
case INPUT_STREAM:
return (DataSource) new InputStreamDataSource(context, dataSpec, inputStream);
case BUFFERED_INPUT_STREAM:
default:
return (DataSource) new InputStreamBufferedDataSource(context, dataSpec, inputStream);
}
};

ExtractorsFactory extractorsFactory = () ->new Extractor[] {new H264Extractor(performancePreset.h264ReaderMaxSyncFrameSize, performancePreset.h264ReaderSampleTime)};
MediaSource mediaSource = new ProgressiveMediaSource.Factory(dataSourceFactory, extractorsFactory).createMediaSource(MediaItem.fromUri(Uri.EMPTY));
mPlayer.setMediaSource(mediaSource);

mPlayer.prepare();
mPlayer.play();
mPlayer.addListener(new ExoPlayer.EventListener() {
@Override
@NonNullApi
public void onPlayerError(ExoPlaybackException error) {
switch (error.type) {
case ExoPlaybackException.TYPE_SOURCE:
Log.e(TAG, "PLAYER_SOURCE - TYPE_SOURCE: " + error.getSourceException().getMessage());
(new Handler(Looper.getMainLooper())).postDelayed(() -> restart(), 1000);
break;
case ExoPlaybackException.TYPE_REMOTE:
Log.e(TAG, "PLAYER_SOURCE - TYPE_REMOTE: " + error.getMessage());
break;
case ExoPlaybackException.TYPE_RENDERER:
Log.e(TAG, "PLAYER_SOURCE - TYPE_RENDERER: " + error.getRendererException().getMessage());
break;
case ExoPlaybackException.TYPE_UNEXPECTED:
Log.e(TAG, "PLAYER_SOURCE - TYPE_UNEXPECTED: " + error.getUnexpectedException().getMessage());
break;
}
}
Log.d(TAG, "preset: " + performancePreset);

@Override
public void onPlaybackStateChanged(@NonNullApi int state) {
switch (state) {
case Player.STATE_IDLE:
case Player.STATE_READY:
case Player.STATE_BUFFERING:
break;
case Player.STATE_ENDED:
Log.d(TAG, "PLAYER_STATE - ENDED");
sendEvent(VideoReaderEventMessageCode.WAITING_FOR_VIDEO); // let MainActivity know so it can hide watermark/show settings button
(new Handler(Looper.getMainLooper())).postDelayed(() -> restart(), 1000);
break;
}
DataSource.Factory dataSourceFactory = () -> {
switch (performancePreset.dataSourceType){
case INPUT_STREAM:
return (DataSource) new InputStreamDataSource(context, dataSpec, inputStream);
case BUFFERED_INPUT_STREAM:
default:
return (DataSource) new InputStreamBufferedDataSource(context, dataSpec, inputStream);
}
};

ExtractorsFactory extractorsFactory = () ->new Extractor[] {new H264Extractor(performancePreset.h264ReaderMaxSyncFrameSize, performancePreset.h264ReaderSampleTime)};
MediaSource mediaSource = new ProgressiveMediaSource.Factory(dataSourceFactory, extractorsFactory).createMediaSource(MediaItem.fromUri(Uri.EMPTY));
mPlayer.setMediaSource(mediaSource);

mPlayer.prepare();
mPlayer.play();
mPlayer.addListener(new ExoPlayer.EventListener() {
@Override
@NonNullApi
public void onPlayerError(ExoPlaybackException error) {
switch (error.type) {
case ExoPlaybackException.TYPE_SOURCE:
Log.e(TAG, "PLAYER_SOURCE - TYPE_SOURCE: " + error.getSourceException().getMessage());
(new Handler(Looper.getMainLooper())).postDelayed(() -> restart(), 1000);
break;
case ExoPlaybackException.TYPE_REMOTE:
Log.e(TAG, "PLAYER_SOURCE - TYPE_REMOTE: " + error.getSourceException().getMessage());
break;
case ExoPlaybackException.TYPE_RENDERER:
Log.e(TAG, "PLAYER_SOURCE - TYPE_RENDERER: " + error.getSourceException().getMessage());
break;
case ExoPlaybackException.TYPE_UNEXPECTED:
Log.e(TAG, "PLAYER_SOURCE - TYPE_UNEXPECTED: " + error.getSourceException().getMessage());
break;
}
});
}

mPlayer.addVideoListener(new VideoListener() {
@Override
public void onRenderedFirstFrame() {
Log.d(TAG, "PLAYER_RENDER - FIRST FRAME");
sendEvent(VideoReaderEventMessageCode.VIDEO_PLAYING); // let MainActivity know so it can hide watermark/show settings button
@Override
public void onPlaybackStateChanged(@NonNullApi int state) {
switch (state) {
case Player.STATE_IDLE:
case Player.STATE_READY:
case Player.STATE_BUFFERING:
break;
case Player.STATE_ENDED:
Log.d(TAG, "PLAYER_STATE - ENDED");
sendEvent(VideoReaderEventMessageCode.WAITING_FOR_VIDEO); // let MainActivity know so it can hide watermark/show settings button
(new Handler(Looper.getMainLooper())).postDelayed(() -> restart(), 1000);
break;
}
}
});

mPlayer.addVideoListener(new VideoListener() {
@Override
public void onRenderedFirstFrame() {
Log.d(TAG, "PLAYER_RENDER - FIRST FRAME");
sendEvent(VideoReaderEventMessageCode.VIDEO_PLAYING); // let MainActivity know so it can hide watermark/show settings button
}

@Override
public void onVideoSizeChanged(int width, int height, int unappliedRotationDegrees, float pixelWidthHeightRatio) {
if (!zoomedIn) {
ConstraintLayout.LayoutParams params = (ConstraintLayout.LayoutParams) surfaceView.getLayoutParams();
params.dimensionRatio = width + ":" + height;
surfaceView.setLayoutParams(params);
}
@Override
public void onVideoSizeChanged(int width, int height, int unappliedRotationDegrees, float pixelWidthHeightRatio) {
if (!zoomedIn) {
ConstraintLayout.LayoutParams params = (ConstraintLayout.LayoutParams) surfaceView.getLayoutParams();
params.dimensionRatio = width + ":" + height;
surfaceView.setLayoutParams(params);
vrView.setAspect(width, height);
}
});
}
});
}

private void sendEvent(VideoReaderEventMessageCode eventCode) {
Expand Down Expand Up @@ -200,6 +229,9 @@ public void restart() {
public void stop() {
if (mPlayer != null)
mPlayer.release();

// VR Related teardown
vrView.stop();
}

public enum VideoReaderEventMessageCode {WAITING_FOR_VIDEO, VIDEO_PLAYING}
Expand Down
Loading