The trick to playing video with effects on Android is to use OpenGL for actual frame display and then use (API 11+) SurfaceTexture class to render video as a texture on screen. See GitHub for full app demo.
The process has two steps:
1. Initialize OpenGL Context to render on a SurfaceTexture
Since there is no SurfaceTexture equivalent of GLSurfaceView we have to initialize an OpenGL context manually. In bottom example we initialize OpenGL ES 2.0 context with disabled depth buffer prepared for 2D video rendering. We also define abstract draw metod to be extended with our drawing logic.
/**
* Renderer which initializes OpenGL 2.0 context on a passed surface and starts a rendering thread
*
* This class has to be subclassed to be used properly
*/
public abstract class TextureSurfaceRenderer implements Runnable
{
private static final int EGL_OPENGL_ES2_BIT = 4;
private static final int EGL_CONTEXT_CLIENT_VERSION = 0x3098;
private static final String LOG_TAG = "SurfaceTest.GL";
protected final SurfaceTexture texture;
private EGL10 egl;
private EGLDisplay eglDisplay;
private EGLContext eglContext;
private EGLSurface eglSurface;
protected int width;
protected int height;
private boolean running;
/**
* @param texture Surface texture on which to render. This has to be called AFTER the texture became available
* @param width Width of the passed surface
* @param height Height of the passed surface
*/
public TextureSurfaceRenderer(SurfaceTexture texture, int width, int height)
{
this.texture = texture;
this.width = width;
this.height = height;
this.running = true;
Thread thrd = new Thread(this);
thrd.start();
}
@Override
public void run()
{
initGL();
initGLComponents();
Log.d(LOG_TAG, "OpenGL init OK.");
while (running)
{
long loopStart = System.currentTimeMillis();
pingFps();
if (draw())
{
egl.eglSwapBuffers(eglDisplay, eglSurface);
}
long waitDelta = 16 - (System.currentTimeMillis() - loopStart); // Targeting 60 fps, no need for faster
if (waitDelta > 0)
{
try
{
Thread.sleep(waitDelta);
}
catch (InterruptedException e)
{
continue;
}
}
}
deinitGLComponents();
deinitGL();
}
/**
* Main draw function, subclass this and add custom drawing code here. The rendering thread will attempt to limit
* FPS to 60 to keep CPU usage low.
*/
protected abstract boolean draw();
/**
* OpenGL component initialization funcion. This is called after OpenGL context has been initialized on the rendering thread.
* Subclass this and initialize shaders / textures / other GL related components here.
*/
protected abstract void initGLComponents();
protected abstract void deinitGLComponents();
private long lastFpsOutput = 0;
private int frames;
private void pingFps()
{
if (lastFpsOutput == 0)
lastFpsOutput = System.currentTimeMillis();
frames ++;
if (System.currentTimeMillis() - lastFpsOutput > 1000)
{
Log.d(LOG_TAG, "FPS: " + frames);
lastFpsOutput = System.currentTimeMillis();
frames = 0;
}
}
/**
* Call when activity pauses. This stops the rendering thread and deinitializes OpenGL.
*/
public void onPause()
{
running = false;
}
private void initGL()
{
egl = (EGL10) EGLContext.getEGL();
eglDisplay = egl.eglGetDisplay(EGL10.EGL_DEFAULT_DISPLAY);
int[] version = new int[2];
egl.eglInitialize(eglDisplay, version);
EGLConfig eglConfig = chooseEglConfig();
eglContext = createContext(egl, eglDisplay, eglConfig);
eglSurface = egl.eglCreateWindowSurface(eglDisplay, eglConfig, texture, null);
if (eglSurface == null || eglSurface == EGL10.EGL_NO_SURFACE)
{
throw new RuntimeException("GL Error: " + GLUtils.getEGLErrorString(egl.eglGetError()));
}
if (!egl.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext))
{
throw new RuntimeException("GL Make current error: " + GLUtils.getEGLErrorString(egl.eglGetError()));
}
}
private void deinitGL()
{
egl.eglMakeCurrent(eglDisplay, EGL10.EGL_NO_SURFACE, EGL10.EGL_NO_SURFACE, EGL10.EGL_NO_CONTEXT);
egl.eglDestroySurface(eglDisplay, eglSurface);
egl.eglDestroyContext(eglDisplay, eglContext);
egl.eglTerminate(eglDisplay);
Log.d(LOG_TAG, "OpenGL deinit OK.");
}
private EGLContext createContext(EGL10 egl, EGLDisplay eglDisplay, EGLConfig eglConfig)
{
int[] attribList = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL10.EGL_NONE };
return egl.eglCreateContext(eglDisplay, eglConfig, EGL10.EGL_NO_CONTEXT, attribList);
}
private EGLConfig chooseEglConfig()
{
int[] configsCount = new int[1];
EGLConfig[] configs = new EGLConfig[1];
int[] configSpec = getConfig();
if (!egl.eglChooseConfig(eglDisplay, configSpec, configs, 1, configsCount))
{
throw new IllegalArgumentException("Failed to choose config: " + GLUtils.getEGLErrorString(egl.eglGetError()));
}
else if (configsCount[0] > 0)
{
return configs[0];
}
return null;
}
private int[] getConfig()
{
return new int[] {
EGL10.EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
EGL10.EGL_RED_SIZE, 8,
EGL10.EGL_GREEN_SIZE, 8,
EGL10.EGL_BLUE_SIZE, 8,
EGL10.EGL_ALPHA_SIZE, 8,
EGL10.EGL_DEPTH_SIZE, 0,
EGL10.EGL_STENCIL_SIZE, 0,
EGL10.EGL_NONE
};
}
@Override
protected void finalize() throws Throwable
{
super.finalize();
running = false;
}
}
When using this class we have to be careful to call onPause for proper cleanup when user leaves the app.
2. Implement video rendering loop
There are two key points to remember when rendering video from MediaPlayer with OpenGL:
- The shader texture type for video has to be of
samplerExternalOEStype - You have to call updateTexImage to actually transfer video frame from HW decoder to the OpenGL texture
- You have to call getTransformMatrix to get rotation flags from video frame and apply it in the shader to get properly rotated video
Basic video renderer class (without aspect ratio adjustment) looks like this:
public class VideoTextureRenderer extends TextureSurfaceRenderer implements SurfaceTexture.OnFrameAvailableListener
{
private static final String vertexShaderCode =
"attribute vec4 vPosition;" +
"attribute vec4 vTexCoordinate;" +
"uniform mat4 textureTransform;" +
"varying vec2 v_TexCoordinate;" +
"void main() {" +
" v_TexCoordinate = (textureTransform * vTexCoordinate).xy;" +
" gl_Position = vPosition;" +
"}";
private static final String fragmentShaderCode =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;" +
"uniform samplerExternalOES texture;" +
"varying vec2 v_TexCoordinate;" +
"void main () {" +
" vec4 color = texture2D(texture, v_TexCoordinate);" +
" gl_FragColor = color;" +
"}";
private static float squareSize = 1.0f;
private static float squareCoords[] = { -squareSize, squareSize, 0.0f, // top left
-squareSize, -squareSize, 0.0f, // bottom left
squareSize, -squareSize, 0.0f, // bottom right
squareSize, squareSize, 0.0f }; // top right
private static short drawOrder[] = { 0, 1, 2, 0, 2, 3};
private Context ctx;
// Texture to be shown in backgrund
private FloatBuffer textureBuffer;
private float textureCoords[] = { 0.0f, 1.0f, 0.0f, 1.0f,
0.0f, 0.0f, 0.0f, 1.0f,
1.0f, 0.0f, 0.0f, 1.0f,
1.0f, 1.0f, 0.0f, 1.0f };
private int[] textures = new int[1];
private int vertexShaderHandle;
private int fragmentShaderHandle;
private int shaderProgram;
private FloatBuffer vertexBuffer;
private ShortBuffer drawListBuffer;
private SurfaceTexture videoTexture;
private float[] videoTextureTransform;
private boolean frameAvailable = false;
private int videoWidth;
private int videoHeight;
private boolean adjustViewport = false;
public VideoTextureRenderer(Context context, SurfaceTexture texture, int width, int height)
{
super(texture, width, height);
this.ctx = context;
videoTextureTransform = new float[16];
}
private void loadShaders()
{
vertexShaderHandle = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER);
GLES20.glShaderSource(vertexShaderHandle, vertexShaderCode);
GLES20.glCompileShader(vertexShaderHandle);
checkGlError("Vertex shader compile");
fragmentShaderHandle = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER);
GLES20.glShaderSource(fragmentShaderHandle, fragmentShaderCode);
GLES20.glCompileShader(fragmentShaderHandle);
checkGlError("Pixel shader compile");
shaderProgram = GLES20.glCreateProgram();
GLES20.glAttachShader(shaderProgram, vertexShaderHandle);
GLES20.glAttachShader(shaderProgram, fragmentShaderHandle);
GLES20.glLinkProgram(shaderProgram);
checkGlError("Shader program compile");
int[] status = new int[1];
GLES20.glGetProgramiv(shaderProgram, GLES20.GL_LINK_STATUS, status, 0);
if (status[0] != GLES20.GL_TRUE) {
String error = GLES20.glGetProgramInfoLog(shaderProgram);
Log.e("SurfaceTest", "Error while linking program:\n" + error);
}
}
private void setupVertexBuffer()
{
// Draw list buffer
ByteBuffer dlb = ByteBuffer.allocateDirect(drawOrder. length * 2);
dlb.order(ByteOrder.nativeOrder());
drawListBuffer = dlb.asShortBuffer();
drawListBuffer.put(drawOrder);
drawListBuffer.position(0);
// Initialize the texture holder
ByteBuffer bb = ByteBuffer.allocateDirect(squareCoords.length * 4);
bb.order(ByteOrder.nativeOrder());
vertexBuffer = bb.asFloatBuffer();
vertexBuffer.put(squareCoords);
vertexBuffer.position(0);
}
private void setupTexture(Context context)
{
ByteBuffer texturebb = ByteBuffer.allocateDirect(textureCoords.length * 4);
texturebb.order(ByteOrder.nativeOrder());
textureBuffer = texturebb.asFloatBuffer();
textureBuffer.put(textureCoords);
textureBuffer.position(0);
// Generate the actual texture
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glGenTextures(1, textures, 0);
checkGlError("Texture generate");
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textures[0]);
checkGlError("Texture bind");
videoTexture = new SurfaceTexture(textures[0]);
videoTexture.setOnFrameAvailableListener(this);
}
@Override
protected boolean draw()
{
synchronized (this)
{
if (frameAvailable)
{
videoTexture.updateTexImage();
videoTexture.getTransformMatrix(videoTextureTransform);
frameAvailable = false;
}
else
{
return false;
}
}
if (adjustViewport)
adjustViewport();
GLES20.glClearColor(1.0f, 0.0f, 0.0f, 0.0f);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
// Draw texture
GLES20.glUseProgram(shaderProgram);
int textureParamHandle = GLES20.glGetUniformLocation(shaderProgram, "texture");
int textureCoordinateHandle = GLES20.glGetAttribLocation(shaderProgram, "vTexCoordinate");
int positionHandle = GLES20.glGetAttribLocation(shaderProgram, "vPosition");
int textureTranformHandle = GLES20.glGetUniformLocation(shaderProgram, "textureTransform");
GLES20.glEnableVertexAttribArray(positionHandle);
GLES20.glVertexAttribPointer(positionHandle, 3, GLES20.GL_FLOAT, false, 4 * 3, vertexBuffer);
GLES20.glBindTexture(GLES20.GL_TEXTURE0, textures[0]);
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glUniform1i(textureParamHandle, 0);
GLES20.glEnableVertexAttribArray(textureCoordinateHandle);
GLES20.glVertexAttribPointer(textureCoordinateHandle, 4, GLES20.GL_FLOAT, false, 0, textureBuffer);
GLES20.glUniformMatrix4fv(textureTranformHandle, 1, false, videoTextureTransform, 0);
GLES20.glDrawElements(GLES20.GL_TRIANGLES, drawOrder.length, GLES20.GL_UNSIGNED_SHORT, drawListBuffer);
GLES20.glDisableVertexAttribArray(positionHandle);
GLES20.glDisableVertexAttribArray(textureCoordinateHandle);
return true;
}
private void adjustViewport()
{
float surfaceAspect = height / (float)width;
float videoAspect = videoHeight / (float)videoWidth;
if (surfaceAspect > videoAspect)
{
float heightRatio = height / (float)videoHeight;
int newWidth = (int)(width * heightRatio);
int xOffset = (newWidth - width) / 2;
GLES20.glViewport(-xOffset, 0, newWidth, height);
}
else
{
float widthRatio = width / (float)videoWidth;
int newHeight = (int)(height * widthRatio);
int yOffset = (newHeight - height) / 2;
GLES20.glViewport(0, -yOffset, width, newHeight);
}
adjustViewport = false;
}
@Override
protected void initGLComponents()
{
setupVertexBuffer();
setupTexture(ctx);
loadShaders();
}
@Override
protected void deinitGLComponents()
{
GLES20.glDeleteTextures(1, textures, 0);
GLES20.glDeleteProgram(shaderProgram);
videoTexture.release();
videoTexture.setOnFrameAvailableListener(null);
}
public void setVideoSize(int width, int height)
{
this.videoWidth = width;
this.videoHeight = height;
adjustViewport = true;
}
public void checkGlError(String op)
{
int error;
while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
Log.e("SurfaceTest", op + ": glError " + GLUtils.getEGLErrorString(error));
}
}
public SurfaceTexture getVideoTexture()
{
return videoTexture;
}
@Override
public void onFrameAvailable(SurfaceTexture surfaceTexture)
{
synchronized (this)
{
frameAvailable = true;
}
}
}
To keep CPU and battery usage down we only render new frame when it's available from the HW decoder (signaled by the SurfaceTextureListener).
Using this class is fairly simple, we initialize MediaPlayer with our SurfaceTexture is available.
private void startPlaying()
{
renderer = new VideoTextureRenderer(this, surface.getSurfaceTexture(), surfaceWidth, surfaceHeight);
player = new MediaPlayer();
try
{
AssetFileDescriptor afd = getAssets().openFd("big_buck_bunny.mp4");
player.setDataSource(afd.getFileDescriptor(), afd.getStartOffset(), afd.getLength());
player.setSurface(new Surface(renderer.getVideoTexture()));
player.setLooping(true);
player.prepare();
renderer.setVideoSize(player.getVideoWidth(), player.getVideoHeight());
player.start();
}
catch (IOException e)
{
throw new RuntimeException("Could not open input video!");
}
}
@Override
public void onSurfaceTextureAvailable(SurfaceTexture surface, int width, int height)
{
surfaceWidth = width;
surfaceHeight = height;
startPlaying();
}
To add effects to playback just change the fragment shader accordingly, e.g. to get a pixelated effect:
private static final String fragmentShaderCode =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;" +
"uniform samplerExternalOES texture;" +
"uniform float pixelationRatio;" +
"uniform float aspectRatio;" +
"varying vec2 v_TexCoordinate;" +
"void main () {" +
" vec2 sampleDivisor = vec2(pixelationRatio, pixelationRatio / aspectRatio); " +
" vec2 samplePos = v_TexCoordinate - mod(v_TexCoordinate, sampleDivisor) + 0.5 * sampleDivisor;" +
" gl_FragColor = texture2D(texture, samplePos);" +
"}";
Full sample application is available on GitHub.