java - Android bytedeco javacpp ffmpeg decode h264 bytes to yuv and render with openGL ES 2.0. Wrong colors -
there! try display video stream, comes server byte array. data in array h264 encoded image , decode bytedeco javacpp-presets library in way:
public class dmdecoder { private static final string log_tag = "dmdecoder"; private avcodec avcodec; private avcodeccontext avcodeccontext; private avframe avframe; private avpacket avpacket; private boolean wasiframe; private long iframetimestampms; private int maxfps; private int codecid; private dmdecodercallback callback; public dmdecoder(dmdecodercallback cb) { this.callback = cb; this.codecid = av_codec_id_h264; avcodec_register_all(); restart(); } public void restart() { stop(); start(); } public void stop() { frames = 0; if (avcodeccontext != null) { avcodec_close(avcodeccontext); avcodec_free_context(avcodeccontext); avcodeccontext = null; } if (avcodec != null) { av_free(avcodec); avcodec = null; } if (avframe != null) { av_frame_free(avframe); avframe = null; } if (avpacket != null) { av_free_packet(avpacket); avpacket = null; } } public void start() { avcodec = avcodec_find_decoder(codecid); avcodeccontext = avcodec_alloc_context3(avcodec); avdictionary opts = new avdictionary(); avcodec_open2(avcodeccontext, avcodec, opts); avframe = av_frame_alloc(); avpacket = new avpacket(); av_init_packet(avpacket); } public videoframe decode(byte[] data, int dataoffset, int datasize) { avpacket.pts(av_nopts_value); avpacket.dts(av_nopts_value); avpacket.data(new bytepointer(data).position(dataoffset)); avpacket.size(datasize); avpacket.pos(-1); intbuffer gotpicture = intbuffer.allocate(1); int processedbytes = avcodec_decode_video2( avcodeccontext, avframe, gotpicture, avpacket); if (avframe.width() == 0 || avframe.height() == 0) return null; videoframe frame = new videoframe(); frame.colorplane0 = new byte[avframe.width() * avframe.height()]; frame.colorplane1 = new byte[avframe.width() / 2 * avframe.height() / 2]; frame.colorplane2 = new byte[avframe.width() / 2 * avframe.height() / 2]; if (avframe.data(0) != null) avframe.data(0).get(frame.colorplane0); if (avframe.data(1) != null) avframe.data(1).get(frame.colorplane1); if (avframe.data(2) != null) avframe.data(2).get(frame.colorplane2); frame.linesize0 = avframe.width(); frame.linesize1 = avframe.width() / 2; frame.linesize2 = avframe.width() / 2; frame.width = avframe.width(); frame.height = avframe.height(); return frame; } } videoframe class simple pojo:
public class videoframe { public byte[] colorplane0; public byte[] colorplane1; public byte[] colorplane2; public int linesize0; public int linesize1; public int linesize2; public int width; public int height; public long presentationtime; } after decoding send frame glrenderer class
public class glrenderer implements glsurfaceview.renderer { private static final string log_tag = "glrenderer"; private textureplane plane; private concurrentlinkedqueue<videoframe> frames; private int maxfps = 30; private videoframe currentframe; private long starttime, endtime; private int viewwidth, viewheight; private boolean isfirstframeprocessed; public glrenderer(int viewwidth, int viewheight) { frames = new concurrentlinkedqueue<>(); this.viewwidth = viewwidth; this.viewheight = viewheight; } // mmvpmatrix abbreviation "model view projection matrix" private final float[] mmvpmatrix = new float[16]; private final float[] mprojectionmatrix = new float[16]; private final float[] mviewmatrix = new float[16]; @override public void onsurfacecreated(gl10 unused, eglconfig config) { // set background frame color gles20.glclearcolor(0.1f, 0.1f, 0.1f, 1.0f); plane = new textureplane(); } public void setmaxfps(int maxfps) { this.maxfps = maxfps; } @override public void ondrawframe(gl10 unused) { // draw background color gles20.glclear(gles20.gl_color_buffer_bit | gles20.gl_depth_buffer_bit); // set camera position (view matrix) matrix.setlookatm(mviewmatrix, 0, 0, 0, -3, 0f, 0f, 0f, 0f, 1.0f, 0.0f); // calculate projection , view transformation matrix.multiplymm(mmvpmatrix, 0, mprojectionmatrix, 0, mviewmatrix, 0); if (!isfirstframeprocessed) checkviewport(viewwidth, viewheight); if (maxfps > 0 && starttime > 0) { endtime = system.currenttimemillis(); long time = endtime - starttime; // long wantedtime = 1000 / maxfps; // long wait; if (time < wantedtime) { wait = wantedtime - time; // try { thread.sleep(wait); } catch (interruptedexception e) { log.e(log_tag, "thread interrupted exception"); } } } starttime = system.currenttimemillis(); tick(); plane.draw(mmvpmatrix); } private void updateframe(videoframe frame) { plane.updatetexture(frame.colorplane0, frame.width, frame.height, 0); plane.updatetexture(frame.colorplane1, frame.width / 2, frame.height / 2, 1); plane.updatetexture(frame.colorplane2, frame.width / 2, frame.height / 2, 2); plane.settexturewidth(frame.width); plane.settextureheight(frame.height); } private void tick() { if (frames.isempty()) return; videoframe frame = frames.peek(); if (frame == null) return; long tms = system.currenttimemillis(); if (frame.presentationtime <= tms) { updateframe(frame); currentframe = frame; frames.remove(frame); } } @override public void onsurfacechanged(gl10 unused, int width, int height) { checkviewport(width, height); viewwidth = width; viewheight = height; plane.settexturewidth(width); plane.settextureheight(height); } private void checkviewport(int width, int height) { float viewratio = (float) width / height; if (currentframe != null) { float targetratio = (float) currentframe.width / currentframe.height; int x, y, newwidth, newheight; if (targetratio > viewratio) { newwidth = width; newheight = (int) (width / targetratio); x = 0; y = (height - newheight) / 2; } else { newheight = height; newwidth = (int) (height * targetratio); y = 0; x = (width - newwidth) / 2; } gles20.glviewport(x, y, newwidth, newheight); } else { gles20.glviewport(0, 0, width, height); } matrix.frustumm(mprojectionmatrix, 0, 1, -1, -1, 1, 3, 4); } public void addframe(videoframe frame) { if (frame != null) { frames.add(frame); } } } glrenderer works simple opengl polygon, on draw textures
public class textureplane { private static final string log_tag = "textureplane"; private final string vertexshadercode = "" + "uniform mat4 umvpmatrix;" + "attribute vec4 vposition;" + "attribute vec2 a_texcoordinate;" + "varying vec2 v_texcoordinate;" + "void main() {" + " gl_position = umvpmatrix * vposition;" + " v_texcoordinate = a_texcoordinate;" + "}"; private final string fragmentshadercode = "" + "precision mediump float;" + "varying vec2 v_texcoordinate;" + "uniform sampler2d s_texture_y;" + "uniform sampler2d s_texture_u;" + "uniform sampler2d s_texture_v;" + "void main() {" + " float y = texture2d(s_texture_y, v_texcoordinate).r;" + " float u = texture2d(s_texture_u, v_texcoordinate).r - 0.5;" + " float v = texture2d(s_texture_v, v_texcoordinate).r - 0.5;" + " float r = y + 1.13983 * v;" + " float g = y - 0.39465 * u - 0.58060 * v;" + " float b = y + 2.03211 * u;" + " gl_fragcolor = vec4(r, g, b, 1.0);" + "}"; private final floatbuffer vertexbuffer; private final floatbuffer texturebuffer; private final shortbuffer drawlistbuffer; private final int mprogram; private int mpositionhandle; private int mmvpmatrixhandle; // number of coordinates per vertex in array private static final int coords_per_vertex = 3; private static final int coords_per_texture = 2; private static float squarecoords[] = { -1f, 1f, 0.0f, -1f, -1f, 0.0f, 1f, -1f, 0.0f, 1f, 1f, 0.0f }; private static float uvs[] = { 0.0f, 0.0f, 0.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f }; private final short draworder[] = {0, 1, 2, 0, 2, 3}; // order draw vertices private final int vertexstride = coords_per_vertex * 4; // 4 bytes per vertex private int texturewidth = 640; private int textureheight = 480; private int ytextureuniformhandle; private int utextureuniformhandle; private int vtextureuniformhandle; private int ytexturehandle; private int utexturehandle; private int vtexturehandle; private int mtexturecoordinatehandle; public void settexturewidth(int texturewidth) { this.texturewidth = texturewidth; } public int gettexturewidth() { return texturewidth; } public void settextureheight(int textureheight) { this.textureheight = textureheight; } public int gettextureheight() { return textureheight; } /** * sets drawing object data use in opengl es context. */ public textureplane() { // initialize vertex byte buffer shape coordinates bytebuffer bb = bytebuffer.allocatedirect(squarecoords.length * 4); bb.order(byteorder.nativeorder()); vertexbuffer = bb.asfloatbuffer(); vertexbuffer.put(squarecoords); vertexbuffer.position(0); // initialize byte buffer draw list bytebuffer dlb = bytebuffer.allocatedirect(draworder.length * 2); dlb.order(byteorder.nativeorder()); drawlistbuffer = dlb.asshortbuffer(); drawlistbuffer.put(draworder); drawlistbuffer.position(0); // initialize byte buffer draw list bytebuffer tbb = bytebuffer.allocatedirect(uvs.length * 4); tbb.order(byteorder.nativeorder()); texturebuffer = tbb.asfloatbuffer(); texturebuffer.put(uvs); texturebuffer.position(0); mprogram = gles20.glcreateprogram(); // create empty opengl program compileshaders(); setuptextures(); } public void setuptextures() { ytexturehandle = setuptexture(null, texturewidth, textureheight, 0); utexturehandle = setuptexture(null, texturewidth, textureheight, 1); vtexturehandle = setuptexture(null, texturewidth, textureheight, 2); } public int setuptexture(bytebuffer data, int width, int height, int index) { final int[] texturehandle = new int[1]; gles20.glgentextures(1, texturehandle, 0); if (texturehandle[0] != 0) { // bind texture in opengl gles20.glactivetexture(gles20.gl_texture0 + index); gles20.glbindtexture(gles20.gl_texture_2d, texturehandle[0]); updatetexture(data, width, height, index); // set filtering gles20.gltexparameterf(gles20.gl_texture_2d, gles20.gl_texture_mag_filter, gles20.gl_linear); gles20.gltexparameterf(gles20.gl_texture_2d, gles20.gl_texture_min_filter, gles20.gl_linear); // set wrapping mode gles20.gltexparameterf(gles20.gl_texture_2d, gles20.gl_texture_wrap_s, gles20.gl_repeat); gles20.gltexparameterf(gles20.gl_texture_2d, gles20.gl_texture_wrap_t, gles20.gl_repeat); } if (texturehandle[0] == 0) { log.e(log_tag, "error loading texture."); } return texturehandle[0]; } public void updatetexture(byte[] data, int width, int height, int index) { if (data == null) { if (width == 0 || height == 0) { width = texturewidth; height = textureheight; } data = new byte[width * height]; if (index == 0) { arrays.fill(data, y); } else if (index == 1) { arrays.fill(data, u); } else { arrays.fill(data, v); } } bytebuffer.wrap(data); bytebuffer.position(0); gles20.glactivetexture(gles20.gl_texture0 + index); gles20.glteximage2d(gles20.gl_texture_2d, 0, gles20.gl_luminance, width, height, 0, gles20.gl_luminance, gles20.gl_unsigned_byte, bytebuffer); } private void compileshaders() { // prepare shaders , opengl program int vertexshader = loadshader( gles20.gl_vertex_shader, vertexshadercode); int fragmentshader = loadshader( gles20.gl_fragment_shader, fragmentshadercode); gles20.glattachshader(mprogram, vertexshader); // add vertex shader program gles20.glattachshader(mprogram, fragmentshader); // add fragment shader program gles20.gllinkprogram(mprogram); // create opengl program executables checkglerror("gllinkprogram"); // add program opengl environment gles20.gluseprogram(mprogram); mpositionhandle = gles20.glgetattriblocation(mprogram, "vposition"); mtexturecoordinatehandle = gles20.glgetattriblocation(mprogram, "a_texcoordinate"); gles20.glenablevertexattribarray(mpositionhandle); gles20.glenablevertexattribarray(mtexturecoordinatehandle); ytextureuniformhandle = gles20.glgetuniformlocation(mprogram, "s_texture_y"); utextureuniformhandle = gles20.glgetuniformlocation(mprogram, "s_texture_u"); vtextureuniformhandle = gles20.glgetuniformlocation(mprogram, "s_texture_v"); mmvpmatrixhandle = gles20.glgetuniformlocation(mprogram, "umvpmatrix"); checkglerror("glgetuniformlocation"); } /** * utility method compiling opengl shader. * <p/> * <p><strong>note:</strong> when developing shaders, use checkglerror() * method debug shader coding errors.</p> * * @param type - vertex or fragment shader type. * @param shadercode - string containing shader code. * @return - returns id shader. */ public int loadshader(int type, string shadercode) { // create vertex shader type (gles20.gl_vertex_shader) // or fragment shader type (gles20.gl_fragment_shader) int shader = gles20.glcreateshader(type); // add source code shader , compile gles20.glshadersource(shader, shadercode); gles20.glcompileshader(shader); return shader; } /** * utility method debugging opengl calls. provide name of call * after making it: * <p/> * <pre> * mcolorhandle = gles20.glgetuniformlocation(mprogram, "vcolor"); * myglrenderer.checkglerror("glgetuniformlocation");</pre> * * if operation not successful, check throws error. * * @param gloperation - name of opengl call check. */ public void checkglerror(string gloperation) { int error; string errorstring; while ((error = gles20.glgeterror()) != gles20.gl_no_error) { errorstring = glu.gluerrorstring(error); string message = gloperation + ": glerror " + error + ": " + errorstring; log.e(log_tag, message); throw new runtimeexception(message); } } public void draw(float[] mvpmatrix) { // prepare triangle coordinate data gles20.glvertexattribpointer( mpositionhandle, coords_per_vertex, gles20.gl_float, false, vertexstride, vertexbuffer); gles20.glvertexattribpointer( mtexturecoordinatehandle, coords_per_texture, gles20.gl_float, false, 0, texturebuffer); gles20.gluniformmatrix4fv(mmvpmatrixhandle, 1, false, mvpmatrix, 0); checkglerror("gluniformmatrix4fv"); gles20.gluniform1i(ytextureuniformhandle, 0); gles20.gluniform1i(utextureuniformhandle, 1); gles20.gluniform1i(vtextureuniformhandle, 2); // draw square gles20.gldrawelements( gles20.gl_triangles, draworder.length, gles20.gl_unsigned_short, drawlistbuffer); } } but have problem there. gl surface display image wrong colors. image
what i'm doing wrong?
update:
as ronald s. bultje say, added glbindtexture(...) function in code. , updatetexture(...) method looks this:
public void updatetexture(byte[] data, int width, int height, int index) { if (data == null) { if (width == 0 || height == 0) { width = texturewidth; height = textureheight; } data = new byte[width * height]; if (index == 0) { arrays.fill(data, y); } else if (index == 1) { arrays.fill(data, u); } else { arrays.fill(data, v); } } bytebuffer.wrap(data); bytebuffer.position(0); gles20.glactivetexture(gles20.gl_texture0 + index); int texturehandle = index == 0 ? ytexturehandle : index == 1 ? utexturehandle : vtexturehandle; gles20.glbindtexture(gles20.gl_texture_2d, texturehandle); gles20.glteximage2d(gles20.gl_texture_2d, 0, gles20.gl_luminance, width, height, 0, gles20.gl_luminance, gles20.gl_unsigned_byte, bytebuffer); }
your updatetexture() function doesn't call gles20.glbindtexture(gles20.gl_texture_2d, texturehandle[index]); after calling gles20.glactivetexture(gles20.gl_texture0 + index);
[edit] given code, index==0?ytexturehandle:index==1?utexturehandle?vtexturehandle, i'm sure can figure out how refactor code make easier.
Comments
Post a Comment