Android How to play video on the surface (OpenGL)

Question

Need help

• How to play videos on Surface(OpenGL) in Android? I tried to play the video in mySurfaceView extends SurfaceView using the setSurface() help method in MediaPlayer .

   SurfaceTexture mTexture = new SurfaceTexture(texture_id); Surface mSurface = new Surface(mTexture); MediaPlayer mp = new MediaPlayer(); mp.setSurface(mSurface); 

I only got audio-video playback that is not playing.

2. How to get a video buffer to send to OpenGL ??
3. How to play video on GLTexture ?

Answer 1

From Android source code ...

 import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.FloatBuffer; import javax.microedition.khronos.egl.EGLConfig; import javax.microedition.khronos.opengles.GL10; import android.content.Context; import android.graphics.SurfaceTexture; import android.media.MediaPlayer; import android.opengl.GLES20; import android.opengl.GLSurfaceView; import android.opengl.Matrix; import android.util.Log; import android.view.Surface; class VideoSurfaceView extends GLSurfaceView { VideoRender mRenderer; private MediaPlayer mMediaPlayer = null; public VideoSurfaceView(Context context, MediaPlayer mp) { super(context); setEGLContextClientVersion(2); mMediaPlayer = mp; mRenderer = new VideoRender(context); setRenderer(mRenderer); } @Override public void onResume() { queueEvent(new Runnable(){ public void run() { mRenderer.setMediaPlayer(mMediaPlayer); }}); super.onResume(); } private static class VideoRender implements GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener { private static String TAG = "VideoRender"; private static final int FLOAT_SIZE_BYTES = 4; private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES; private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0; private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3; private final float[] mTriangleVerticesData = { // X, Y, Z, U, V -1.0f, -1.0f, 0, 0.f, 0.f, 1.0f, -1.0f, 0, 1.f, 0.f, -1.0f, 1.0f, 0, 0.f, 1.f, 1.0f, 1.0f, 0, 1.f, 1.f, }; private FloatBuffer mTriangleVertices; private final String mVertexShader = "uniform mat4 uMVPMatrix;\n" + "uniform mat4 uSTMatrix;\n" + "attribute vec4 aPosition;\n" + "attribute vec4 aTextureCoord;\n" + "varying vec2 vTextureCoord;\n" + "void main() {\n" + " gl_Position = uMVPMatrix * aPosition;\n" + " vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" + "}\n"; private final String mFragmentShader = "#extension GL_OES_EGL_image_external : require\n" + "precision mediump float;\n" + "varying vec2 vTextureCoord;\n" + "uniform samplerExternalOES sTexture;\n" + "void main() {\n" + " gl_FragColor = texture2D(sTexture, vTextureCoord);\n" + "}\n"; private float[] mMVPMatrix = new float[16]; private float[] mSTMatrix = new float[16]; private int mProgram; private int mTextureID; private int muMVPMatrixHandle; private int muSTMatrixHandle; private int maPositionHandle; private int maTextureHandle; private SurfaceTexture mSurface; private boolean updateSurface = false; private static int GL_TEXTURE_EXTERNAL_OES = 0x8D65; private MediaPlayer mMediaPlayer; public VideoRender(Context context) { mTriangleVertices = ByteBuffer.allocateDirect( mTriangleVerticesData.length * FLOAT_SIZE_BYTES) .order(ByteOrder.nativeOrder()).asFloatBuffer(); mTriangleVertices.put(mTriangleVerticesData).position(0); Matrix.setIdentityM(mSTMatrix, 0); } public void setMediaPlayer(MediaPlayer player) { mMediaPlayer = player; } public void onDrawFrame(GL10 glUnused) { synchronized(this) { if (updateSurface) { mSurface.updateTexImage(); mSurface.getTransformMatrix(mSTMatrix); updateSurface = false; } } GLES20.glClearColor(0.0f, 1.0f, 0.0f, 1.0f); GLES20.glClear( GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT); GLES20.glUseProgram(mProgram); checkGlError("glUseProgram"); GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID); mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET); GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false, TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices); checkGlError("glVertexAttribPointer maPosition"); GLES20.glEnableVertexAttribArray(maPositionHandle); checkGlError("glEnableVertexAttribArray maPositionHandle"); mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET); GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false, TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices); checkGlError("glVertexAttribPointer maTextureHandle"); GLES20.glEnableVertexAttribArray(maTextureHandle); checkGlError("glEnableVertexAttribArray maTextureHandle"); Matrix.setIdentityM(mMVPMatrix, 0); GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0); GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); checkGlError("glDrawArrays"); GLES20.glFinish(); } public void onSurfaceChanged(GL10 glUnused, int width, int height) { } public void onSurfaceCreated(GL10 glUnused, EGLConfig config) { mProgram = createProgram(mVertexShader, mFragmentShader); if (mProgram == 0) { return; } maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition"); checkGlError("glGetAttribLocation aPosition"); if (maPositionHandle == -1) { throw new RuntimeException("Could not get attrib location for aPosition"); } maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord"); checkGlError("glGetAttribLocation aTextureCoord"); if (maTextureHandle == -1) { throw new RuntimeException("Could not get attrib location for aTextureCoord"); } muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix"); checkGlError("glGetUniformLocation uMVPMatrix"); if (muMVPMatrixHandle == -1) { throw new RuntimeException("Could not get attrib location for uMVPMatrix"); } muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix"); checkGlError("glGetUniformLocation uSTMatrix"); if (muSTMatrixHandle == -1) { throw new RuntimeException("Could not get attrib location for uSTMatrix"); } int[] textures = new int[1]; GLES20.glGenTextures(1, textures, 0); mTextureID = textures[0]; GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID); checkGlError("glBindTexture mTextureID"); GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST); GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); /* * Create the SurfaceTexture that will feed this textureID, * and pass it to the MediaPlayer */ mSurface = new SurfaceTexture(mTextureID); mSurface.setOnFrameAvailableListener(this); Surface surface = new Surface(mSurface); mMediaPlayer.setSurface(surface); surface.release(); try { mMediaPlayer.prepare(); } catch (IOException t) { Log.e(TAG, "media player prepare failed"); } synchronized(this) { updateSurface = false; } mMediaPlayer.start(); } synchronized public void onFrameAvailable(SurfaceTexture surface) { updateSurface = true; } private int loadShader(int shaderType, String source) { int shader = GLES20.glCreateShader(shaderType); if (shader != 0) { GLES20.glShaderSource(shader, source); GLES20.glCompileShader(shader); int[] compiled = new int[1]; GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0); if (compiled[0] == 0) { Log.e(TAG, "Could not compile shader " + shaderType + ":"); Log.e(TAG, GLES20.glGetShaderInfoLog(shader)); GLES20.glDeleteShader(shader); shader = 0; } } return shader; } private int createProgram(String vertexSource, String fragmentSource) { int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource); if (vertexShader == 0) { return 0; } int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource); if (pixelShader == 0) { return 0; } int program = GLES20.glCreateProgram(); if (program != 0) { GLES20.glAttachShader(program, vertexShader); checkGlError("glAttachShader"); GLES20.glAttachShader(program, pixelShader); checkGlError("glAttachShader"); GLES20.glLinkProgram(program); int[] linkStatus = new int[1]; GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0); if (linkStatus[0] != GLES20.GL_TRUE) { Log.e(TAG, "Could not link program: "); Log.e(TAG, GLES20.glGetProgramInfoLog(program)); GLES20.glDeleteProgram(program); program = 0; } } return program; } private void checkGlError(String op) { int error; while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) { Log.e(TAG, op + ": glError " + error); throw new RuntimeException(op + ": glError " + error); } } } // End of class VideoRender. } // End of class VideoSurfaceView. 

Answer 2

I think you can’t. At least this is what I learned. My plan was to have some kind of OpenGL scene (text ticker) while playing the video. Since Android uses HW decoding to display video, this will not be done with OpenGL. I also tried playing videos in OpenGL using ffmpeg, but I found out that no device I was trying to work with was powerful enough to do SW decoding through ffmpeg.

So I had to use VideoView to display my video and put GLSurfaceView on top of it to view the text of my code. But you have to make GLSurfaceView translucent, like in ApiDemos, using the "TransucentGLSurfaceViewActivity".

Another thing that I realized: if you install GLSurfaceView on top of VideoView, your fps will drop dramatically from 60 fps (opengl) to 30-40 frames per second. This applies to all versions of Android version 2.x. Last week I had the opportunity to test it on Android 4, and this time I did not have a gap in fps. Perhaps they really improved the graphics pipeline for ICS.

Cheers, -chris -

Answer 3

 mMediaPlayer.setSurface(new Surface(mSurfaceTexture)); 

You can use the above line of code to use it in your mediaPlayerObject over your desired surface texture, which is an applied texture on top of your surface view.

Hope this helps.

Answer 4

I just converted Java to Kotlin version

 internal inline fun <T> glRun(message: String = "", block: (() -> T)): T { return block().also { var error: Int = GLES20.glGetError() while (error != GLES20.GL_NO_ERROR) { error = GLES20.glGetError() Log.d("MOVIE_GL_ERROR", "$message: $error") throw RuntimeException("GL Error: $message") } } } class MovieRenderer: GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener { private var program = 0 private var textureId = 0 // Handles private var mvpMatrixHandle = 0 private var stMatrixHandle = 0 private var positionHandle = 0 private var textureHandle = 0 // Surface Texture private var updateSurface = false private lateinit var surfaceTexture: SurfaceTexture // Matrices private var mvpMatrix = FloatArray(16) private var stMatrix = FloatArray(16) // float buffer private val vertices: FloatBuffer = ByteBuffer.allocateDirect(VERTICES_DATA.size * FLOAT_SIZE_BYTES) .order(ByteOrder.nativeOrder()) .asFloatBuffer().also { it.put(VERTICES_DATA).position(0) } var mediaPlayer: MediaPlayer? = null @Synchronized override fun onFrameAvailable(surfaceTexture: SurfaceTexture?) { updateSurface = true } override fun onDrawFrame(gl: GL10?) { synchronized(this) { if (updateSurface) { surfaceTexture.updateTexImage() surfaceTexture.getTransformMatrix(stMatrix) updateSurface = false } } GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f) GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT or GLES20.GL_COLOR_BUFFER_BIT) glRun("glUseProgram: $program") { GLES20.glUseProgram(program) } vertices.position(VERTICES_POS_OFFSET); glRun("glVertexAttribPointer: Stride bytes") { GLES20.glVertexAttribPointer(positionHandle, 3, GLES20.GL_FLOAT, false, VERTICES_STRIDE_BYTES, vertices) } glRun("glEnableVertexAttribArray") { GLES20.glEnableVertexAttribArray(positionHandle) } vertices.position(VERTICES_UV_OFFSET) glRun("glVertexAttribPointer: texture handle") { GLES20.glVertexAttribPointer(textureHandle, 3, GLES20.GL_FLOAT, false, VERTICES_STRIDE_BYTES, vertices) } glRun("glEnableVertexAttribArray") { GLES20.glEnableVertexAttribArray(textureHandle) } Matrix.setIdentityM(mvpMatrix, 0) glRun("glUniformMatrix4fv: mvpMatrix") { GLES20.glUniformMatrix4fv(mvpMatrixHandle, 1, false, mvpMatrix, 0) } glRun("glUniformMatrix4fv: stMatrix") { GLES20.glUniformMatrix4fv(stMatrixHandle, 1, false, stMatrix, 0) } glRun("glDrawArrays: GL_TRIANGLE_STRIP") { GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4) } GLES20.glFinish() } override fun onSurfaceChanged(gl: GL10?, width: Int, height: Int) { GLES20.glViewport(0, 0, width, height) } override fun onSurfaceCreated(gl: GL10?, config: EGLConfig?) { program = createProgram() positionHandle = "aPosition".attr() textureHandle = "aTextureCoord".attr() mvpMatrixHandle = "uMVPMatrix".uniform() stMatrixHandle = "uSTMatrix".uniform() createTexture() } private fun createTexture() { val textures = IntArray(1) GLES20.glGenTextures(1, textures, 0) textureId = textures.first() glRun("glBindTexture textureId") { GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, textureId) } GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST) GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR) surfaceTexture = SurfaceTexture(textureId) surfaceTexture.setOnFrameAvailableListener(this) val surface = Surface(surfaceTexture) mediaPlayer?.setSurface(surface) surface.release() try { mediaPlayer?.prepare() } catch (error: IOException) { Log.e("MovieRenderer", "media player prepare failed"); throw error } synchronized(this) { updateSurface = false } mediaPlayer?.start() } private fun String.attr(): Int { return glRun("Get attribute location: $this") { GLES20.glGetAttribLocation(program, this).also { if (it == -1) fail("Error Attribute: $this not found!") } } } private fun String.uniform(): Int { return glRun("Get uniform location: $this") { GLES20.glGetUniformLocation(program, this).also { if (it == -1) fail("Error Uniform: $this not found!") } } } companion object { private const val GL_TEXTURE_EXTERNAL_OES = 0x8D65 private const val FLOAT_SIZE_BYTES = 4 private const val VERTICES_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES private const val VERTICES_POS_OFFSET = 0 private const val VERTICES_UV_OFFSET = 3 private val VERTICES_DATA = floatArrayOf( -1.0f, -1.0f, 0f, 0.0f, 0.0f, 1.0f, -1.0f, 0f, 1.0f, 0.0f, -1.0f, 1.0f, 0f, 0.0f, 1.0f, 1.0f, 1.0f, 0f, 1.0f, 1.0f ) private const val VERTEX_SHADER = """ uniform mat4 uMVPMatrix; uniform mat4 uSTMatrix; attribute vec4 aPosition; attribute vec4 aTextureCoord; varying vec2 vTextureCoord; void main() { gl_Position = uMVPMatrix * aPosition; vTextureCoord = (uSTMatrix * aTextureCoord).xy; } """ private const val FRAGMENT_SHADER = """ #extension GL_OES_EGL_image_external : require precision mediump float; varying vec2 vTextureCoord; uniform samplerExternalOES sTexture; void main() { gl_FragColor = texture2D(sTexture, vTextureCoord); } """ private fun createShader(type: Int, source: String): Int { val shader = GLES20.glCreateShader(type) if (shader == 0) throw RuntimeException("Cannot create shader $type\n$source") GLES20.glShaderSource(shader, source) GLES20.glCompileShader(shader) val args = IntArray(1) GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, args, 0) if (args.first() == 0) { Log.e("MOVIE_SHADER", "Failed to compile shader source") Log.e("MOVIE_SHADER", GLES20.glGetShaderInfoLog(shader)) GLES20.glDeleteShader(shader) throw RuntimeException("Could not compile shader $source\n$type") } return shader } private fun createProgram(vertexShaderSource: String = VERTEX_SHADER, fragmentShaderSource: String = FRAGMENT_SHADER): Int { val vertexShader = createShader(GLES20.GL_VERTEX_SHADER, vertexShaderSource) val fragmentShader = createShader(GLES20.GL_FRAGMENT_SHADER, fragmentShaderSource) val program = GLES20.glCreateProgram() if (program == 0) throw RuntimeException("Cannot create program") glRun("Attach vertex shader to program") { GLES20.glAttachShader(program, vertexShader) } glRun("Attach fragment shader to program") { GLES20.glAttachShader(program, fragmentShader) } GLES20.glLinkProgram(program) val args = IntArray(1) GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, args, 0) if (args.first() != GLES20.GL_TRUE) { val info = GLES20.glGetProgramInfoLog(program) GLES20.glDeleteProgram(program) throw RuntimeException("Cannot link program $program, Info: $info") } return program } private fun fail(message: String): Nothing { throw RuntimeException(message) } } }