Android 视频画面实时文字化图像

一、前言

在本篇之前，很多博客已经实现过图片文本化，但是由于渲染方式的不合理，大部分设计都做不到尽可能实时播放，本篇将在已有的基础上进行一些优化，使得视频文字化具备一定的实时性。

下图总体上视频和文字化的画面基本是实时的，上面是SurfaceView，下面是我们自定义的View类，通过实时抓帧然后实时转bitmap，做到了基本同步。

二、现状

目前很多流行的方式是修改像素的色值，这个性能差距太大，导致卡顿非常严重，无法做到实时性。当然也有通过open gl mask实现的，但是在贴图这一块我们知道，open gl只支持绘制三角、点和线，因此"文字"纹理生成还得利用Canvas实现。

但对于对帧率要求不高的需求，是不是有更好的方案呢？

三、优化方案

优化点1: 使用Shader

网上很多博客都是利用Bitmap#getPixel和Bitmap#setPixel进行，这个计算量显然太大了，就算使用open gl 也未必好，因此首先解决的问题就是使用Shader着色。

优化点2: 提前计算好单个文字所占的最大空间

显然这个原因是更加整齐的排列文字，其次也可以做到降低计算量和提高灵活度

优化点3:使用队列

对于了编解码的开发而言，使用队列不仅可以复用buffer，而且还能提高绘制性能，另外必要时可以丢帧。

基于以上三点，基本可以做到实时字符化画面，当然，我们这里是彩色的，对于灰度图的需求，可通过设置Paint的ColorMatrix实现，总之，要避免遍历修改像素了RGB。

四、关键代码

使用shader着色

 this.bitmapShader = new BitmapShader(inputBitmap, Shader.TileMode.CLAMP, Shader.TileMode.CLAMP);
 this.mCharPaint.setShader(bitmapShader);
//用下面方式清空bitmap
 boardBitmap.bitmap.eraseColor(Color.TRANSPARENT);

计算字符size

    private Rect computeMaxCharWidth(TextPaint drawerPaint, String text) {
        if (TextUtils.isEmpty(text)) {
            return null;
        }
        Rect result = new Rect(); // 文字所在区域的矩形
        Rect md = new Rect();
        for (int i = 0; i < text.length(); i++) {
            String s = text.charAt(i) + "";
            if(TextUtils.isEmpty(s)) {
                continue;
            }
            drawerPaint.getTextBounds(s, 0, 1, md);
            if (md.width() > result.width()) {
                result.right = md.width();
            }
            if (md.height() > result.height()) {
                result.bottom = md.height();
            }
        }
        return result;
    }

定义双队列，实现控制和享元机制

    private BitmapPool bitmapPool = new BitmapPool();
    private BitmapPool recyclePool = new BitmapPool();
    
    static class BitmapPool {
        int width;
        int height;
        private LinkedBlockingQueue<BitmapItem> linkedBlockingQueue = new LinkedBlockingQueue<>(5);
    }

    static class BitmapItem{
        Bitmap bitmap;
        boolean isUsed = false;
    }

完整代码

public class WordBitmapView extends View {
    private final DisplayMetrics mDM;
    private TextPaint mCharPaint;
    private TextPaint mDrawerPaint = null;
    private Bitmap inputBitmap;
    private Rect charMxWidth = null ;
    private String text = "a1b2c3d4e5f6h7j8k9l0";
    private float textBaseline;
    private BitmapShader bitmapShader;
    public WordBitmapView(Context context) {
        this(context, null);
    }
    public WordBitmapView(Context context, AttributeSet attrs) {
        this(context, attrs, 0);
    }
    public WordBitmapView(Context context, AttributeSet attrs, int defStyleAttr) {
        super(context, attrs, defStyleAttr);
        mDM = getResources().getDisplayMetrics();
        initPaint();
    }

    @Override
    protected void onMeasure(int widthMeasureSpec, int heightMeasureSpec) {
        super.onMeasure(widthMeasureSpec, heightMeasureSpec);

        int widthMode = MeasureSpec.getMode(widthMeasureSpec);
        int widthSize = MeasureSpec.getSize(widthMeasureSpec);

        if (widthMode != MeasureSpec.EXACTLY) {
            widthSize = mDM.widthPixels / 2;
        }

        int heightMode = MeasureSpec.getMode(heightMeasureSpec);
        int heightSize = MeasureSpec.getSize(heightMeasureSpec);

        if (heightMode != MeasureSpec.EXACTLY) {
            heightSize = widthSize / 2;
        }
        setMeasuredDimension(widthSize, heightSize);

        textBaseline = getTextPaintBaseline(mDrawerPaint);
    }


    public float dp2px(float dp) {
        return TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, dp, mDM);
    }

    public float sp2px(float dp) {
        return TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_SP, dp, mDM);
    }

    @Override
    protected void onSizeChanged(int w, int h, int oldw, int oldh) {
        super.onSizeChanged(w, h, oldw, oldh);
        recyclePool.clear();
        bitmapPool.clear();
    }

    Matrix matrix = new Matrix();
    @Override
    protected void onDraw(Canvas canvas) {
        super.onDraw(canvas);
        int width = getWidth();
        int height = getHeight();
        if (width <= 1 || height <= 1) {
            return;
        }
        BitmapItem bitmapItem = bitmapPool.linkedBlockingQueue.poll();
        if (bitmapItem == null || inputBitmap == null) {
            return;
        }
        if(!bitmapItem.isUsed){
            return;
        }
        canvas.drawBitmap(bitmapItem.bitmap,matrix,mDrawerPaint);
        bitmapItem.isUsed = false;
        try {
            recyclePool.linkedBlockingQueue.offer(bitmapItem,16,TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

    public static float getTextPaintBaseline(Paint p) {
        Paint.FontMetrics fontMetrics = p.getFontMetrics();
        return (fontMetrics.descent - fontMetrics.ascent) / 2 - fontMetrics.descent;
    }

    private Rect computeMaxCharWidth(TextPaint drawerPaint, String text) {
        if (TextUtils.isEmpty(text)) {
            return null;
        }
        Rect result = new Rect(); // 文字所在区域的矩形
        Rect md = new Rect();
        for (int i = 0; i < text.length(); i++) {
            String s = text.charAt(i) + "";
            if(TextUtils.isEmpty(s)) {
                continue;
            }
            drawerPaint.getTextBounds(s, 0, 1, md);
            if (md.width() > result.width()) {
                result.right = md.width();
            }
            if (md.height() > result.height()) {
                result.bottom = md.height();
            }
        }
        return result;
    }

    private BitmapPool bitmapPool = new BitmapPool();
    private BitmapPool recyclePool = new BitmapPool();

    static class BitmapPool {
        int width;
        int height;
        private LinkedBlockingQueue<BitmapItem> linkedBlockingQueue = new LinkedBlockingQueue<>(5);
        public void clear(){
            Iterator<BitmapItem> iterator = linkedBlockingQueue.iterator();
            do{
                if(!iterator.hasNext()) break;
                BitmapItem next = iterator.next();
                if(!next.bitmap.isRecycled()) {
                    next.bitmap.recycle();
                }
                iterator.remove();
            }while (true);
        }

        public int getWidth() {
            return width;
        }

        public int getHeight() {
            return height;
        }

        public void setHeight(int height) {
            this.height = height;
        }

        public void setWidth(int width) {
            this.width = width;
        }
    }

    class BitmapItem{
        Bitmap bitmap;
        boolean isUsed = false;
    }


   //视频图片入队
    public void queueInputBitmap(Bitmap inputBitmap) {
        this.inputBitmap = inputBitmap;

        if(charMxWidth  == null){
            charMxWidth = computeMaxCharWidth(mDrawerPaint,text);
        }
        if(charMxWidth == null || charMxWidth.width() == 0){
            return;
        }

        if(this.bitmapPool != null && this.inputBitmap != null){
            if(this.bitmapPool.getWidth() != this.inputBitmap.getWidth()){
                bitmapPool.clear();
                recyclePool.clear();
            }else if(this.bitmapPool.getHeight() != this.inputBitmap.getHeight()){
                bitmapPool.clear();
                recyclePool.clear();
            }
        }
        bitmapPool.setWidth(inputBitmap.getWidth());
        bitmapPool.setHeight(inputBitmap.getHeight());
        recyclePool.setWidth(inputBitmap.getWidth());
        recyclePool.setHeight(inputBitmap.getHeight());

        BitmapItem boardBitmap = recyclePool.linkedBlockingQueue.poll();
        if (boardBitmap == null && inputBitmap != null) {
            boardBitmap = new BitmapItem();
            boardBitmap.bitmap = Bitmap.createBitmap(inputBitmap.getWidth(), inputBitmap.getHeight(), Bitmap.Config.ARGB_8888);
        }
        boardBitmap.isUsed = true;
        int bitmapWidth = inputBitmap.getWidth();
        int bitmapHeight = inputBitmap.getHeight();
        int unitWidth = (int) (charMxWidth.width()  *1.5);
        int unitHeight = charMxWidth.height()  + 2;
        int centerY = charMxWidth.centerY();
        float hLineCharNum = bitmapWidth * 1F / unitWidth;
        float vLineCharNum = bitmapHeight * 1F / unitHeight;


        this.bitmapShader = new BitmapShader(inputBitmap, Shader.TileMode.CLAMP, Shader.TileMode.CLAMP);
        this.mCharPaint.setShader(bitmapShader);

        boardBitmap.bitmap.eraseColor(Color.TRANSPARENT);

        Canvas drawCanvas = new Canvas(boardBitmap.bitmap);
        int k = (int) (Math.random() * text.length());
        for (int i = 0; i < vLineCharNum; i++) {
            for (int j = 0; j < hLineCharNum; j++) {
                int length = text.length();
                int x = unitWidth * j;
                int y = centerY + i * unitHeight;
                String c = text.charAt(k % length) + "";
                drawCanvas.drawText(c, x, y + textBaseline, mCharPaint);
                k++;
            }
        }
        try {
            bitmapPool.linkedBlockingQueue.offer(boardBitmap,16, TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        postInvalidate();

    }
    private void initPaint() {
        // 实例化画笔并打开抗锯齿
        mCharPaint = new TextPaint(Paint.ANTI_ALIAS_FLAG);
        mCharPaint.setAntiAlias(true);
        mCharPaint.setStyle(Paint.Style.FILL);
        mCharPaint.setStrokeCap(Paint.Cap.ROUND);
        mDrawerPaint = new TextPaint(Paint.ANTI_ALIAS_FLAG);
        mDrawerPaint.setAntiAlias(true);
        mDrawerPaint.setStyle(Paint.Style.FILL);
        mDrawerPaint.setStrokeCap(Paint.Cap.ROUND);
    }


}

五、总结

Android中Shader是非常重要的工具，我们无需单独修改像素的情况下就能实现快速渲染字符，得意与Shader出色的渲染能力。另外由于时间原因，这里对字符的绘制并没有做到很精确，仅仅选了一些比较中规中列的排列，后续再继续完善吧。