feat: 图像识别、协程步骤优化

Author: Ven

assists/src/main/AndroidManifest.xml

@@ -12,7 +12,7 @@
                 android:resource="@xml/assists_provider_paths" />
         </provider>
         <service
-            android:name="com.ven.assists.ScreenService"
+            android:name="com.ven.assists.ScreenCaptureService"
             android:enabled="true"
             android:exported="false"
             android:foregroundServiceType="mediaProjection" />

assists/src/main/java/com/ven/assists/Assists.kt

@@ -24,10 +24,16 @@ import com.blankj.utilcode.util.ActivityUtils
 import com.blankj.utilcode.util.LogUtils
 import com.blankj.utilcode.util.ScreenUtils
 import com.blankj.utilcode.util.ThreadUtils
+import com.ven.assists.stepper.ScreenCaptureAutoEnable
+import com.ven.assists.stepper.Step
+import com.ven.assists.stepper.StepCollector
 import com.ven.assists.stepper.StepManager
+import kotlinx.coroutines.CompletableDeferred
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.withContext
 
 object Assists {
 
@@ -40,6 +46,8 @@ object Assists {
     val screenRequestLaunchers: HashMap<Activity, ActivityResultLauncher<Intent>> = hashMapOf()
 
     private var job = Job()
+
+    //协程域
     var coroutine: CoroutineScope = CoroutineScope(job + Dispatchers.IO)
         private set
         get() {
@@ -117,15 +125,16 @@ object Assists {
                     .createScreenCaptureIntent()
             )
             if (isAutoEnable && service != null) {
+                StepManager.execute(ScreenCaptureAutoEnable::class.java, 1)
             }
         }
     }
 
     /**
      * 是否拥有录屏权限
      */
-    fun isOwnScreenCapture(): Boolean {
-        return screenCaptureService == null
+    fun isEnableScreenCapture(): Boolean {
+        return screenCaptureService != null
     }
 
     fun init(application: Application) {
@@ -193,6 +202,20 @@ object Assists {
         return arrayListOf()
     }
 
+    /**
+     * 判断元素是否包含指定的文本
+     */
+    fun AccessibilityNodeInfo?.containsText(text: String): Boolean {
+        if (this == null) return false
+        getText()?.let {
+            if (it.contains(text)) return true
+        }
+        contentDescription?.let {
+            if (it.contains(text)) return true
+        }
+        return false
+    }
+
 
     /**
      * 根据类型查找元素
@@ -285,17 +308,17 @@ object Assists {
      * @return 执行手势动作总耗时
      */
     @JvmStatic
-    fun gesture(
+    suspend fun gesture(
         startLocation: FloatArray,
         endLocation: FloatArray,
         startTime: Long,
         duration: Long,
-    ): Long {
+    ) {
         gestureListeners.forEach { it.onGestureBegin(startLocation, endLocation) }
         val path = Path()
         path.moveTo(startLocation[0], startLocation[1])
         path.lineTo(endLocation[0], endLocation[1])
-        return gesture(path, startTime, duration)
+        gesture(path, startTime, duration)
     }
 
     /**
@@ -306,34 +329,35 @@ object Assists {
      * @return 执行手势动作总耗时
      */
     @JvmStatic
-    fun gesture(
+    suspend fun gesture(
         path: Path,
         startTime: Long,
         duration: Long,
-    ): Long {
+    ) {
         val builder = GestureDescription.Builder()
         val strokeDescription = GestureDescription.StrokeDescription(path, startTime, duration)
         val gestureDescription = builder.addStroke(strokeDescription).build()
-        object : AccessibilityService.GestureResultCallback() {
+        val deferred = CompletableDeferred<Int>()
+        service?.dispatchGesture(gestureDescription, object : AccessibilityService.GestureResultCallback() {
             override fun onCompleted(gestureDescription: GestureDescription) {
-                gestureListeners.forEach { it.onGestureCompleted() }
-                gestureListeners.forEach { it.onGestureEnd() }
+                deferred.complete(1)
             }
 
             override fun onCancelled(gestureDescription: GestureDescription) {
+                deferred.complete(0)
                 gestureListeners.forEach { it.onGestureCancelled() }
                 gestureListeners.forEach { it.onGestureEnd() }
             }
-        }.apply {
-            if (gestureBeginDelay == 0L) {
-                service?.dispatchGesture(gestureDescription, this, null)
-            } else {
-                ThreadUtils.runOnUiThreadDelayed({
-                    service?.dispatchGesture(gestureDescription, this, null)
-                }, gestureBeginDelay)
-            }
+        }, null) ?: let {
+            deferred.complete(0)
+        }
+        val result = deferred.await()
+        if (result == 1) {
+            gestureListeners.forEach { it.onGestureCompleted() }
+        } else {
+            gestureListeners.forEach { it.onGestureCancelled() }
         }
-        return gestureBeginDelay + startTime + duration
+        gestureListeners.forEach { it.onGestureEnd() }
     }
 
     /**
@@ -348,9 +372,9 @@ object Assists {
     /**
      * 手势点击元素所处的位置
      */
-    fun AccessibilityNodeInfo.gestureClick(): Long {
+    suspend fun AccessibilityNodeInfo.gestureClick() {
         val rect = getBoundsInScreen()
-        return gestureClick(rect.left + 15F, rect.top + 15F, 10)
+        gestureClick(rect.left + 15F, rect.top + 15F, 10)
     }
 
     /**
@@ -363,21 +387,21 @@ object Assists {
     /**
      * 手势长按元素所处的位置
      */
-    fun AccessibilityNodeInfo.gestureLongClick(): Long {
+    suspend fun AccessibilityNodeInfo.gestureLongClick() {
         val rect = getBoundsInScreen()
-        return gestureClick(rect.left + 15F, rect.top + 15F, 1000)
+        gestureClick(rect.left + 15F, rect.top + 15F, 1000)
     }
 
     /**
      * 点击屏幕指定位置
      * @return 执行手势动作总耗时
      */
-    fun gestureClick(
+    suspend fun gestureClick(
         x: Float,
         y: Float,
         duration: Long = 10
-    ): Long {
-        return gesture(
+    ) {
+        gesture(
             floatArrayOf(x, y), floatArrayOf(x, y),
             0,
             duration,

assists/src/main/java/com/ven/assists/AssistsWindowLayout.kt

@@ -85,7 +85,7 @@ class AssistsWindowLayout @JvmOverloads constructor(
 
     init {
 
-        setBackgroundColor(Color.parseColor("#80000000"))
+        setBackgroundColor(Color.parseColor("#4D000000"))
 
         assistsWindowLayoutWrapperBinding = AssistsWindowLayoutWrapperBinding.inflate(LayoutInflater.from(getContext()), this).apply {
             ivMove.setOnTouchListener(onTouchMoveListener)

assists/src/main/java/com/ven/assists/AssistsWindowManager.kt

@@ -9,6 +9,7 @@ import android.view.View
 import android.view.ViewGroup
 import android.view.WindowManager
 import androidx.core.view.isInvisible
+import androidx.core.view.isVisible
 import java.util.Collections
 
 object AssistsWindowManager {
@@ -48,13 +49,22 @@ object AssistsWindowManager {
         }
     }
 
+    fun showLastView() {
+        viewList.lastOrNull()?.let {
+            it.view.isVisible = true
+        }
+        switchTouchableAll()
+    }
+
 
-    fun addView(view: View?, params: ViewGroup.LayoutParams) {
+    fun addView(view: View?, params: ViewGroup.LayoutParams, isStack: Boolean = false) {
         view ?: return
-        viewList.forEach {
-            it.view.isInvisible = true
-            if (it.view is AssistsWindowLayout) {
-                it.view.switchNotTouchable()
+        if (!isStack) {
+            viewList.forEach {
+                it.view.isInvisible = true
+                if (it.view is AssistsWindowLayout) {
+                    it.view.switchNotTouchable()
+                }
             }
         }
 

assists/src/main/java/com/ven/assists/OpencvWrapper.kt

@@ -1,21 +1,26 @@
-package com.ven.assists
+package com.ven.assists
 
 import android.graphics.Bitmap
 import android.graphics.BitmapFactory
+import android.util.Log
 import com.blankj.utilcode.util.ActivityUtils
 import com.blankj.utilcode.util.LogUtils
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.GlobalScope
+import com.blankj.utilcode.util.PathUtils
 import kotlinx.coroutines.launch
-import kotlinx.coroutines.withContext
 import org.opencv.android.OpenCVLoader
 import org.opencv.android.Utils
 import org.opencv.core.Core
+import org.opencv.core.CvType
 import org.opencv.core.Mat
 import org.opencv.core.Point
+import org.opencv.core.Rect
+import org.opencv.core.Scalar
 import org.opencv.imgproc.Imgproc
+import java.io.FileOutputStream
 import java.io.IOException
 import java.io.InputStream
+import kotlin.math.abs
+
 
 object OpencvWrapper {
 
@@ -29,22 +34,111 @@ object OpencvWrapper {
         }
     }
 
-    fun matchTemplateFromScreen(image: Mat, template: Mat): Point {
+    fun matchTemplate(image: Mat?, template: Mat?, mask: Mat? = null): Mat? {
+        image ?: return null
+        template ?: return null
+        val resultCols: Int = image.cols() - template.cols() + 1
+        val resultRows: Int = image.rows() - template.rows() + 1
+        val result = Mat(resultRows, resultCols, CvType.CV_32FC1)
+        val method = Imgproc.TM_CCORR_NORMED
+        if (mask == null) {
+            Imgproc.matchTemplate(image, template, result, method)
+
+        } else {
+            Imgproc.matchTemplate(image, template, result, method, mask)
+
+        }
+        return result
+    }
+
+    fun getResultWithThreshold(
+        result: Mat,
+        threshold: Double,
+        ignoreX: Double = -1.0,
+        ignoreY: Double = -1.0,
+    ): ArrayList<Point> {
+        val resultList = arrayListOf<Point>()
+        for (y in 0 until result.rows()) {
+            for (x in 0 until result.cols()) {
+                val matchValue = result[y, x]
+                if (matchValue[0] >= threshold) {
+                    val point = Point(x.toDouble(), y.toDouble())
+                    if (resultList.isEmpty() || (ignoreX == -1.0 && ignoreY == -1.0)) {
+                        resultList.add(point)
+                    } else {
+                        var ignore = false
+                        for (value in resultList) {
+                            val ignoreValueX = abs(point.x - value.x)
+                            val ignoreValueY = abs(point.y - value.y)
+                            if (ignoreX != -1.0 && ignoreValueX < ignoreX) {
+                                ignore = true
+                                break
+                            }
+                            if (ignoreY != -1.0 && ignoreValueY < ignoreY) {
+                                ignore = true
+                                break
+                            }
+                        }
+                        if (!ignore) {
+                            resultList.add(point)
+                        }
+                    }
+                }
+            }
+        }
+        return resultList
+    }
+
+    fun matchTemplateFromScreenToMinMaxLoc(image: Mat?, template: Mat?, mask: Mat? = null): Core.MinMaxLocResult? {
+        image ?: return null
+        template ?: return null
         val result = Mat()
-        val method = Imgproc.TM_CCOEFF_NORMED
-        Imgproc.matchTemplate(image, template, result, method)
+        val method = Imgproc.TM_CCORR_NORMED
+        Imgproc.matchTemplate(image, template, result, method, mask)
         val minMaxLocResult = Core.minMaxLoc(result)
-        return minMaxLocResult.maxLoc
+        return minMaxLocResult
     }
 
-    fun getScreen() {
+
+    /**
+     * 创建掩膜
+     */
+    fun createMask(
+        source: Mat,
+        lowerGreen: Scalar,
+        upperGreen: Scalar,
+        requisiteExtraRectList: List<Rect> = arrayListOf(),
+        redundantExtraRectList: List<Rect> = arrayListOf()
+    ): Mat {
+        val hsvImage = Mat()
+        Imgproc.cvtColor(source, hsvImage, Imgproc.COLOR_BGR2HSV)
+        val mask = Mat()
+        Core.inRange(hsvImage, lowerGreen, upperGreen, mask)
+        requisiteExtraRectList.forEach {
+            Imgproc.rectangle(mask, it, Scalar(0.0), -1)
+        }
+        redundantExtraRectList.forEach {
+            Imgproc.rectangle(mask, it, Scalar(255.0), -1)
+
+        }
+        return mask
+    }
+
+    /**
+     * 获取屏幕图像
+     */
+    fun getScreen(): Mat {
         val screenBitmap = Assists.screenCaptureService?.toBitmap()
         val screenMat = Mat()
         Utils.bitmapToMat(screenBitmap, screenMat)
         Imgproc.cvtColor(screenMat, screenMat, Imgproc.COLOR_RGBA2BGR)
+        return screenMat
     }
 
-    fun getTemplateFromAsset(assetPath: String): Mat? {
+    /**
+     * 从Assets获取图像
+     */
+    fun getTemplateFromAssets(assetPath: String): Mat? {
         val bitmap = getBitmapFromAsset(assetPath)
         bitmap ?: return null
         val mat = Mat()
@@ -73,5 +167,12 @@ object OpencvWrapper {
         }
     }
 
+    class MinMaxLocResultWrapper(val minMaxLocResult: Core.MinMaxLocResult, val targetMat: Mat?) {
 
+    }
+
+
+    class ResultWrapper(val result: Mat, val targetMat: Mat?) {
+
+    }
 }