Feature: Interaction with screen element

app/schemas/io.github.sds100.keymapper.data.db.AppDatabase/14.json

@@ -0,0 +1,214 @@
+{
+  "formatVersion": 1,
+  "database": {
+    "version": 14,
+    "identityHash": "ea7690c067f077386e05257595ff45bb",
+    "entities": [
+      {
+        "tableName": "keymaps",
+        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `trigger` TEXT NOT NULL, `action_list` TEXT NOT NULL, `constraint_list` TEXT NOT NULL, `constraint_mode` INTEGER NOT NULL, `flags` INTEGER NOT NULL, `folder_name` TEXT, `is_enabled` INTEGER NOT NULL, `uid` TEXT NOT NULL)",
+        "fields": [
+          {
+            "fieldPath": "id",
+            "columnName": "id",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "trigger",
+            "columnName": "trigger",
+            "affinity": "TEXT",
+            "notNull": true
+          },
+          {
+            "fieldPath": "actionList",
+            "columnName": "action_list",
+            "affinity": "TEXT",
+            "notNull": true
+          },
+          {
+            "fieldPath": "constraintList",
+            "columnName": "constraint_list",
+            "affinity": "TEXT",
+            "notNull": true
+          },
+          {
+            "fieldPath": "constraintMode",
+            "columnName": "constraint_mode",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "flags",
+            "columnName": "flags",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "folderName",
+            "columnName": "folder_name",
+            "affinity": "TEXT",
+            "notNull": false
+          },
+          {
+            "fieldPath": "isEnabled",
+            "columnName": "is_enabled",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "uid",
+            "columnName": "uid",
+            "affinity": "TEXT",
+            "notNull": true
+          }
+        ],
+        "primaryKey": {
+          "autoGenerate": true,
+          "columnNames": [
+            "id"
+          ]
+        },
+        "indices": [],
+        "foreignKeys": []
+      },
+      {
+        "tableName": "fingerprintmaps",
+        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `action_list` TEXT NOT NULL, `constraint_list` TEXT NOT NULL, `constraint_mode` INTEGER NOT NULL, `extras` TEXT NOT NULL, `flags` INTEGER NOT NULL, `is_enabled` INTEGER NOT NULL, PRIMARY KEY(`id`))",
+        "fields": [
+          {
+            "fieldPath": "id",
+            "columnName": "id",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "actionList",
+            "columnName": "action_list",
+            "affinity": "TEXT",
+            "notNull": true
+          },
+          {
+            "fieldPath": "constraintList",
+            "columnName": "constraint_list",
+            "affinity": "TEXT",
+            "notNull": true
+          },
+          {
+            "fieldPath": "constraintMode",
+            "columnName": "constraint_mode",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "extras",
+            "columnName": "extras",
+            "affinity": "TEXT",
+            "notNull": true
+          },
+          {
+            "fieldPath": "flags",
+            "columnName": "flags",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "isEnabled",
+            "columnName": "is_enabled",
+            "affinity": "INTEGER",
+            "notNull": true
+          }
+        ],
+        "primaryKey": {
+          "autoGenerate": false,
+          "columnNames": [
+            "id"
+          ]
+        },
+        "indices": [],
+        "foreignKeys": []
+      },
+      {
+        "tableName": "log",
+        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `time` INTEGER NOT NULL, `severity` INTEGER NOT NULL, `message` TEXT NOT NULL)",
+        "fields": [
+          {
+            "fieldPath": "id",
+            "columnName": "id",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "time",
+            "columnName": "time",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "severity",
+            "columnName": "severity",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "message",
+            "columnName": "message",
+            "affinity": "TEXT",
+            "notNull": true
+          }
+        ],
+        "primaryKey": {
+          "autoGenerate": true,
+          "columnNames": [
+            "id"
+          ]
+        },
+        "indices": [],
+        "foreignKeys": []
+      },
+      {
+        "tableName": "viewids",
+        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `view_id` TEXT NOT NULL, `package_name` TEXT NOT NULL, `full_name` TEXT NOT NULL)",
+        "fields": [
+          {
+            "fieldPath": "id",
+            "columnName": "id",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "viewId",
+            "columnName": "view_id",
+            "affinity": "TEXT",
+            "notNull": true
+          },
+          {
+            "fieldPath": "packageName",
+            "columnName": "package_name",
+            "affinity": "TEXT",
+            "notNull": true
+          },
+          {
+            "fieldPath": "fullName",
+            "columnName": "full_name",
+            "affinity": "TEXT",
+            "notNull": true
+          }
+        ],
+        "primaryKey": {
+          "autoGenerate": true,
+          "columnNames": [
+            "id"
+          ]
+        },
+        "indices": [],
+        "foreignKeys": []
+      }
+    ],
+    "views": [],
+    "setupQueries": [
+      "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
+      "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'ea7690c067f077386e05257595ff45bb')"
+    ]
+  }
+}

app/src/main/java/io/github/sds100/keymapper/KeyMapperApp.kt

@@ -9,6 +9,7 @@ import androidx.lifecycle.OnLifecycleEvent
 import androidx.lifecycle.ProcessLifecycleOwner
 import androidx.multidex.MultiDexApplication
 import com.google.android.material.color.DynamicColors
+import io.github.sds100.keymapper.system.ui.RecordUiElementsController
 import io.github.sds100.keymapper.data.Keys
 import io.github.sds100.keymapper.data.entities.LogEntryEntity
 import io.github.sds100.keymapper.logging.KeyMapperLoggingTree
@@ -142,6 +143,10 @@ class KeyMapperApp : MultiDexApplication() {
         RecordTriggerController(appCoroutineScope, accessibilityServiceAdapter)
     }
 
+    val recordUiElementsController by lazy {
+        RecordUiElementsController(appCoroutineScope, accessibilityServiceAdapter)
+    }
+
     val autoGrantPermissionController by lazy {
         AutoGrantPermissionController(
             appCoroutineScope,

app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt

@@ -12,7 +12,9 @@ import io.github.sds100.keymapper.data.repositories.PreferenceRepository
 import io.github.sds100.keymapper.data.repositories.RoomFingerprintMapRepository
 import io.github.sds100.keymapper.data.repositories.RoomKeyMapRepository
 import io.github.sds100.keymapper.data.repositories.RoomLogRepository
+import io.github.sds100.keymapper.data.repositories.RoomViewIdRepository
 import io.github.sds100.keymapper.data.repositories.SettingsPreferenceRepository
+import io.github.sds100.keymapper.data.repositories.ViewIdRepository
 import io.github.sds100.keymapper.logging.LogRepository
 import io.github.sds100.keymapper.mappings.fingerprintmaps.FingerprintMapRepository
 import io.github.sds100.keymapper.shizuku.ShizukuAdapter
@@ -122,6 +124,20 @@ object ServiceLocator {
         }
     }
 
+    @Volatile
+    private var IRoomViewIdRepository: ViewIdRepository? = null
+
+    fun viewIdRepository(context: Context): ViewIdRepository {
+        synchronized(this) {
+            return IRoomViewIdRepository ?: RoomViewIdRepository(
+                database(context).viewIdDao(),
+                (context.applicationContext as KeyMapperApp).appCoroutineScope,
+            ).also {
+                this.IRoomViewIdRepository = it
+            }
+        }
+    }
+
     @Volatile
     private var backupManager: BackupManager? = null
 
@@ -138,7 +154,7 @@ object ServiceLocator {
             roomKeymapRepository(context),
             settingsRepository(context),
             fingerprintMapRepository(context),
-            soundsManager(context)
+            soundsManager(context),
         ).also {
             this.backupManager = it
         }
@@ -299,7 +315,8 @@ object ServiceLocator {
             AppDatabase.MIGRATION_9_10,
             AppDatabase.MIGRATION_10_11,
             AppDatabase.RoomMigration_11_12(context.applicationContext.legacyFingerprintMapDataStore),
-            AppDatabase.MIGRATION_12_13
+            AppDatabase.MIGRATION_12_13,
+            AppDatabase.MIGRATION_13_14
         ).build()
     }
 }

app/src/main/java/io/github/sds100/keymapper/UseCases.kt

@@ -33,6 +33,8 @@ import io.github.sds100.keymapper.system.inputmethod.KeyMapperImeMessengerImpl
 import io.github.sds100.keymapper.system.inputmethod.ShowInputMethodPickerUseCase
 import io.github.sds100.keymapper.system.inputmethod.ShowInputMethodPickerUseCaseImpl
 import io.github.sds100.keymapper.system.inputmethod.ToggleCompatibleImeUseCaseImpl
+import io.github.sds100.keymapper.system.ui.DisplayUiElementsUseCase
+import io.github.sds100.keymapper.system.ui.DisplayUiElementsUseCaseImpl
 
 /**
  * Created by sds100 on 03/03/2021.
@@ -73,6 +75,13 @@ object UseCases {
         )
     }
 
+    fun displayUiElements(ctx: Context): DisplayUiElementsUseCase {
+        return DisplayUiElementsUseCaseImpl(
+            ServiceLocator.viewIdRepository(ctx),
+            displayPackages(ctx)
+        )
+    }
+
     fun getActionError(ctx: Context) = GetActionErrorUseCaseImpl(
         ServiceLocator.packageManagerAdapter(ctx),
         ServiceLocator.inputMethodAdapter(ctx),

app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt

@@ -1,5 +1,6 @@
 package io.github.sds100.keymapper.actions
 
+import io.github.sds100.keymapper.actions.uielementinteraction.INTERACTIONTYPE
 import io.github.sds100.keymapper.system.camera.CameraLens
 import io.github.sds100.keymapper.system.display.Orientation
 import io.github.sds100.keymapper.system.intents.IntentTarget
@@ -320,6 +321,19 @@ sealed class ActionData {
         override val id = ActionId.SWIPE_SCREEN
     }
 
+    @Serializable
+    data class InteractWithScreenElement(
+        val elementId: String,
+        val packageName: String,
+        val fullName: String,
+        val appName: String?,
+        val onlyIfVisible: Boolean,
+        val interactiontype: INTERACTIONTYPE,
+        val description: String?,
+    ): ActionData() {
+        override val id = ActionId.INTERACT_WITH_SCREEN_ELEMENT
+    }
+
     @Serializable
     data class PhoneCall(
         val number: String