diff --git a/app/schemas/io.github.sds100.keymapper.data.db.AppDatabase/14.json b/app/schemas/io.github.sds100.keymapper.data.db.AppDatabase/14.json new file mode 100644 index 0000000000..33cf7371b6 --- /dev/null +++ b/app/schemas/io.github.sds100.keymapper.data.db.AppDatabase/14.json @@ -0,0 +1,214 @@ +{ + "formatVersion": 1, + "database": { + "version": 14, + "identityHash": "ea7690c067f077386e05257595ff45bb", + "entities": [ + { + "tableName": "keymaps", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `trigger` TEXT NOT NULL, `action_list` TEXT NOT NULL, `constraint_list` TEXT NOT NULL, `constraint_mode` INTEGER NOT NULL, `flags` INTEGER NOT NULL, `folder_name` TEXT, `is_enabled` INTEGER NOT NULL, `uid` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "trigger", + "columnName": "trigger", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "actionList", + "columnName": "action_list", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintList", + "columnName": "constraint_list", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintMode", + "columnName": "constraint_mode", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "flags", + "columnName": "flags", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "folderName", + "columnName": "folder_name", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "isEnabled", + "columnName": "is_enabled", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "uid", + "columnName": "uid", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "fingerprintmaps", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `action_list` TEXT NOT NULL, `constraint_list` TEXT NOT NULL, `constraint_mode` INTEGER NOT NULL, `extras` TEXT NOT NULL, `flags` INTEGER NOT NULL, `is_enabled` INTEGER NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "actionList", + "columnName": "action_list", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintList", + "columnName": "constraint_list", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintMode", + "columnName": "constraint_mode", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "extras", + "columnName": "extras", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "flags", + "columnName": "flags", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isEnabled", + "columnName": "is_enabled", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `time` INTEGER NOT NULL, `severity` INTEGER NOT NULL, `message` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "time", + "columnName": "time", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "severity", + "columnName": "severity", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "message", + "columnName": "message", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "viewids", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `view_id` TEXT NOT NULL, `package_name` TEXT NOT NULL, `full_name` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "viewId", + "columnName": "view_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "package_name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "fullName", + "columnName": "full_name", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [], + "foreignKeys": [] + } + ], + "views": [], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'ea7690c067f077386e05257595ff45bb')" + ] + } +} \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/KeyMapperApp.kt b/app/src/main/java/io/github/sds100/keymapper/KeyMapperApp.kt index bf86727341..0e75689376 100644 --- a/app/src/main/java/io/github/sds100/keymapper/KeyMapperApp.kt +++ b/app/src/main/java/io/github/sds100/keymapper/KeyMapperApp.kt @@ -9,6 +9,7 @@ import androidx.lifecycle.OnLifecycleEvent import androidx.lifecycle.ProcessLifecycleOwner import androidx.multidex.MultiDexApplication import com.google.android.material.color.DynamicColors +import io.github.sds100.keymapper.system.ui.RecordUiElementsController import io.github.sds100.keymapper.data.Keys import io.github.sds100.keymapper.data.entities.LogEntryEntity import io.github.sds100.keymapper.logging.KeyMapperLoggingTree @@ -142,6 +143,10 @@ class KeyMapperApp : MultiDexApplication() { RecordTriggerController(appCoroutineScope, accessibilityServiceAdapter) } + val recordUiElementsController by lazy { + RecordUiElementsController(appCoroutineScope, accessibilityServiceAdapter) + } + val autoGrantPermissionController by lazy { AutoGrantPermissionController( appCoroutineScope, diff --git a/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt b/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt index 0abbd88cd5..8d04ebafa7 100755 --- a/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt +++ b/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt @@ -12,7 +12,9 @@ import io.github.sds100.keymapper.data.repositories.PreferenceRepository import io.github.sds100.keymapper.data.repositories.RoomFingerprintMapRepository import io.github.sds100.keymapper.data.repositories.RoomKeyMapRepository import io.github.sds100.keymapper.data.repositories.RoomLogRepository +import io.github.sds100.keymapper.data.repositories.RoomViewIdRepository import io.github.sds100.keymapper.data.repositories.SettingsPreferenceRepository +import io.github.sds100.keymapper.data.repositories.ViewIdRepository import io.github.sds100.keymapper.logging.LogRepository import io.github.sds100.keymapper.mappings.fingerprintmaps.FingerprintMapRepository import io.github.sds100.keymapper.shizuku.ShizukuAdapter @@ -122,6 +124,20 @@ object ServiceLocator { } } + @Volatile + private var IRoomViewIdRepository: ViewIdRepository? = null + + fun viewIdRepository(context: Context): ViewIdRepository { + synchronized(this) { + return IRoomViewIdRepository ?: RoomViewIdRepository( + database(context).viewIdDao(), + (context.applicationContext as KeyMapperApp).appCoroutineScope, + ).also { + this.IRoomViewIdRepository = it + } + } + } + @Volatile private var backupManager: BackupManager? = null @@ -138,7 +154,7 @@ object ServiceLocator { roomKeymapRepository(context), settingsRepository(context), fingerprintMapRepository(context), - soundsManager(context) + soundsManager(context), ).also { this.backupManager = it } @@ -299,7 +315,8 @@ object ServiceLocator { AppDatabase.MIGRATION_9_10, AppDatabase.MIGRATION_10_11, AppDatabase.RoomMigration_11_12(context.applicationContext.legacyFingerprintMapDataStore), - AppDatabase.MIGRATION_12_13 + AppDatabase.MIGRATION_12_13, + AppDatabase.MIGRATION_13_14 ).build() } } \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/UseCases.kt b/app/src/main/java/io/github/sds100/keymapper/UseCases.kt index 3c6f773263..3e105881f9 100644 --- a/app/src/main/java/io/github/sds100/keymapper/UseCases.kt +++ b/app/src/main/java/io/github/sds100/keymapper/UseCases.kt @@ -33,6 +33,8 @@ import io.github.sds100.keymapper.system.inputmethod.KeyMapperImeMessengerImpl import io.github.sds100.keymapper.system.inputmethod.ShowInputMethodPickerUseCase import io.github.sds100.keymapper.system.inputmethod.ShowInputMethodPickerUseCaseImpl import io.github.sds100.keymapper.system.inputmethod.ToggleCompatibleImeUseCaseImpl +import io.github.sds100.keymapper.system.ui.DisplayUiElementsUseCase +import io.github.sds100.keymapper.system.ui.DisplayUiElementsUseCaseImpl /** * Created by sds100 on 03/03/2021. @@ -73,6 +75,13 @@ object UseCases { ) } + fun displayUiElements(ctx: Context): DisplayUiElementsUseCase { + return DisplayUiElementsUseCaseImpl( + ServiceLocator.viewIdRepository(ctx), + displayPackages(ctx) + ) + } + fun getActionError(ctx: Context) = GetActionErrorUseCaseImpl( ServiceLocator.packageManagerAdapter(ctx), ServiceLocator.inputMethodAdapter(ctx), diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt index d99b76b1cd..e80b0b356c 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt @@ -1,5 +1,6 @@ package io.github.sds100.keymapper.actions +import io.github.sds100.keymapper.actions.uielementinteraction.INTERACTIONTYPE import io.github.sds100.keymapper.system.camera.CameraLens import io.github.sds100.keymapper.system.display.Orientation import io.github.sds100.keymapper.system.intents.IntentTarget @@ -320,6 +321,19 @@ sealed class ActionData { override val id = ActionId.SWIPE_SCREEN } + @Serializable + data class InteractWithScreenElement( + val elementId: String, + val packageName: String, + val fullName: String, + val appName: String?, + val onlyIfVisible: Boolean, + val interactiontype: INTERACTIONTYPE, + val description: String?, + ): ActionData() { + override val id = ActionId.INTERACT_WITH_SCREEN_ELEMENT + } + @Serializable data class PhoneCall( val number: String diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt index f43d06b9b1..37906cf69a 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt @@ -1,5 +1,6 @@ package io.github.sds100.keymapper.actions +import io.github.sds100.keymapper.actions.uielementinteraction.INTERACTIONTYPE import io.github.sds100.keymapper.data.entities.ActionEntity import io.github.sds100.keymapper.data.entities.Extra import io.github.sds100.keymapper.data.entities.getData @@ -14,6 +15,7 @@ import io.github.sds100.keymapper.util.success import io.github.sds100.keymapper.util.then import io.github.sds100.keymapper.util.valueOrNull import splitties.bitflags.hasFlag +import timber.log.Timber /** * Created by sds100 on 13/03/2021. @@ -30,6 +32,7 @@ object ActionDataEntityMapper { ActionEntity.Type.URL -> ActionId.URL ActionEntity.Type.TAP_COORDINATE -> ActionId.TAP_SCREEN ActionEntity.Type.SWIPE_COORDINATE -> ActionId.SWIPE_SCREEN + ActionEntity.Type.INTERACT_WITH_SCREEN_ELEMENT -> ActionId.INTERACT_WITH_SCREEN_ELEMENT ActionEntity.Type.INTENT -> ActionId.INTENT ActionEntity.Type.PHONE_CALL -> ActionId.PHONE_CALL ActionEntity.Type.SOUND -> ActionId.SOUND @@ -104,12 +107,12 @@ object ActionDataEntityMapper { ActionId.SWIPE_SCREEN -> { val splitData = entity.data.trim().split(',') - var xStart = 0 - var yStart = 0 - var xEnd = 0 - var yEnd = 0 - var fingerCount = 1 - var duration = 250 + var xStart = 0; + var yStart = 0; + var xEnd = 0; + var yEnd = 0; + var fingerCount = 1; + var duration = 250; if (splitData.isNotEmpty()) { xStart = splitData[0].trim().toInt() @@ -149,6 +152,30 @@ object ActionDataEntityMapper { ) } + ActionId.INTERACT_WITH_SCREEN_ELEMENT -> { + val splitData = entity.data.trim().split(',') + + val elementId = splitData[0] + val packageName = splitData[1] + val fullName = splitData[2] + val appName = splitData[3] + val onlyIfVisible = splitData[4].toBoolean() + val interactiontype = splitData[5] + + val description = entity.extras.getData(ActionEntity.EXTRA_ELEMENT_DESCRIPTION) + .valueOrNull() + + ActionData.InteractWithScreenElement( + elementId = elementId, + packageName = packageName, + fullName = fullName, + appName = appName, + onlyIfVisible = onlyIfVisible, + interactiontype = INTERACTIONTYPE.valueOf(interactiontype), + description = description + ) + } + ActionId.INTENT -> { val target = entity.extras.getData(ActionEntity.EXTRA_INTENT_TARGET).then { INTENT_TARGET_MAP.getKey(it).success() @@ -210,7 +237,6 @@ object ActionDataEntityMapper { ActionId.VOLUME_TOGGLE_MUTE -> ActionData.Volume.ToggleMute( showVolumeUi ) - ActionId.VOLUME_UNMUTE -> ActionData.Volume.UnMute(showVolumeUi) ActionId.VOLUME_MUTE -> ActionData.Volume.Mute(showVolumeUi) @@ -274,22 +300,16 @@ object ActionDataEntityMapper { when (actionId) { ActionId.PAUSE_MEDIA_PACKAGE -> ActionData.ControlMediaForApp.Pause(packageName) - ActionId.PLAY_MEDIA_PACKAGE -> ActionData.ControlMediaForApp.Play(packageName) - ActionId.PLAY_PAUSE_MEDIA_PACKAGE -> ActionData.ControlMediaForApp.PlayPause(packageName) - ActionId.NEXT_TRACK_PACKAGE -> ActionData.ControlMediaForApp.NextTrack(packageName) - ActionId.PREVIOUS_TRACK_PACKAGE -> ActionData.ControlMediaForApp.PreviousTrack(packageName) - ActionId.FAST_FORWARD_PACKAGE -> ActionData.ControlMediaForApp.FastForward(packageName) - ActionId.REWIND_PACKAGE -> ActionData.ControlMediaForApp.Rewind(packageName) @@ -424,6 +444,7 @@ object ActionDataEntityMapper { is ActionData.PhoneCall -> ActionEntity.Type.PHONE_CALL is ActionData.TapScreen -> ActionEntity.Type.TAP_COORDINATE is ActionData.SwipeScreen -> ActionEntity.Type.SWIPE_COORDINATE + is ActionData.InteractWithScreenElement -> ActionEntity.Type.INTERACT_WITH_SCREEN_ELEMENT is ActionData.Text -> ActionEntity.Type.TEXT_BLOCK is ActionData.Url -> ActionEntity.Type.URL is ActionData.Sound -> ActionEntity.Type.SOUND @@ -464,6 +485,7 @@ object ActionDataEntityMapper { is ActionData.PhoneCall -> data.number is ActionData.TapScreen -> "${data.x},${data.y}" is ActionData.SwipeScreen -> "${data.xStart},${data.yStart},${data.xEnd},${data.yEnd},${data.fingerCount},${data.duration}" + is ActionData.InteractWithScreenElement -> "${data.elementId},${data.packageName},${data.fullName},${data.appName},${data.onlyIfVisible},${data.interactiontype}" is ActionData.Text -> data.text is ActionData.Url -> data.url is ActionData.Sound -> data.soundUid @@ -549,7 +571,6 @@ object ActionDataEntityMapper { else -> emptyList() } - is ActionData.TapScreen -> sequence { if (!data.description.isNullOrBlank()) { yield(Extra(ActionEntity.EXTRA_COORDINATE_DESCRIPTION, data.description)) @@ -562,6 +583,12 @@ object ActionDataEntityMapper { } }.toList() + is ActionData.InteractWithScreenElement -> sequence { + if (!data.description.isNullOrBlank()) { + yield(Extra(ActionEntity.EXTRA_ELEMENT_DESCRIPTION, data.description)) + } + }.toList() + is ActionData.Text -> emptyList() is ActionData.Url -> emptyList() @@ -723,7 +750,7 @@ object ActionDataEntityMapper { ActionId.DISMISS_MOST_RECENT_NOTIFICATION to "dismiss_most_recent_notification", ActionId.DISMISS_ALL_NOTIFICATIONS to "dismiss_all_notifications", - + ActionId.ANSWER_PHONE_CALL to "answer_phone_call", ActionId.END_PHONE_CALL to "end_phone_call" ) diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionId.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionId.kt index b5d0c5de7b..3c41515aa1 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionId.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionId.kt @@ -10,6 +10,7 @@ enum class ActionId { KEY_EVENT, TAP_SCREEN, SWIPE_SCREEN, + INTERACT_WITH_SCREEN_ELEMENT, TEXT, URL, INTENT, diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt index 999368dd0b..6ba09f94c9 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt @@ -40,6 +40,7 @@ object ActionUtils { ActionId.KEY_EVENT -> ActionCategory.INPUT ActionId.TAP_SCREEN -> ActionCategory.INPUT ActionId.SWIPE_SCREEN -> ActionCategory.INPUT + ActionId.INTERACT_WITH_SCREEN_ELEMENT -> ActionCategory.INPUT ActionId.TEXT -> ActionCategory.INPUT ActionId.OPEN_VOICE_ASSISTANT -> ActionCategory.APPS @@ -257,6 +258,7 @@ object ActionUtils { ActionId.KEY_EVENT -> R.string.action_input_key_event ActionId.TAP_SCREEN -> R.string.action_tap_screen ActionId.SWIPE_SCREEN -> R.string.action_swipe_screen + ActionId.INTERACT_WITH_SCREEN_ELEMENT -> R.string.action_interact_with_screen_element ActionId.TEXT -> R.string.action_input_text ActionId.URL -> R.string.action_open_url ActionId.INTENT -> R.string.action_send_intent @@ -367,6 +369,7 @@ object ActionUtils { ActionId.KEY_EVENT -> R.drawable.ic_q_24 ActionId.TAP_SCREEN -> R.drawable.ic_outline_touch_app_24 ActionId.SWIPE_SCREEN -> R.drawable.ic_outline_touch_app_24 + ActionId.INTERACT_WITH_SCREEN_ELEMENT -> R.drawable.ic_outline_interact_with_screen_element_app_24 ActionId.TEXT -> R.drawable.ic_outline_short_text_24 ActionId.URL -> R.drawable.ic_outline_link_24 ActionId.INTENT -> null @@ -604,6 +607,7 @@ fun ActionData.isEditable(): Boolean = when (this) { is ActionData.Flashlight.Disable, is ActionData.TapScreen, is ActionData.SwipeScreen, + is ActionData.InteractWithScreenElement, is ActionData.Text, is ActionData.Url, is ActionData.PhoneCall, diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/BaseActionUiHelper.kt b/app/src/main/java/io/github/sds100/keymapper/actions/BaseActionUiHelper.kt index 6269d526e1..d69a1d5443 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/BaseActionUiHelper.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/BaseActionUiHelper.kt @@ -24,19 +24,19 @@ import splitties.bitflags.hasFlag */ abstract class BaseActionUiHelper, A : Action>( - displayActionUseCase: DisplayActionUseCase, - resourceProvider: ResourceProvider -) : ActionUiHelper, - ResourceProvider by resourceProvider, + displayActionUseCase: DisplayActionUseCase, resourceProvider: ResourceProvider +) : ActionUiHelper, ResourceProvider by resourceProvider, DisplayActionUseCase by displayActionUseCase { override fun getTitle(action: ActionData, showDeviceDescriptors: Boolean): String = when (action) { - is ActionData.App -> - getAppName(action.packageName).handle( - onSuccess = { getString(R.string.description_open_app, it) }, - onError = { getString(R.string.description_open_app, action.packageName) } + is ActionData.App -> getAppName(action.packageName).handle(onSuccess = { + getString( + R.string.description_open_app, + it ) + }, + onError = { getString(R.string.description_open_app, action.packageName) }) is ActionData.AppShortcut -> action.shortcutTitle @@ -75,8 +75,7 @@ abstract class BaseActionUiHelper, A : Action>( val nameToShow = if (showDeviceDescriptors) { InputDeviceUtils.appendDeviceDescriptorToName( - action.device.descriptor, - name + action.device.descriptor, name ) } else { name @@ -98,16 +97,14 @@ abstract class BaseActionUiHelper, A : Action>( is ActionData.DoNotDisturb.Enable -> { val dndModeString = getString(DndModeUtils.getLabel(action.dndMode)) getString( - R.string.action_enable_dnd_mode_formatted, - dndModeString + R.string.action_enable_dnd_mode_formatted, dndModeString ) } is ActionData.DoNotDisturb.Toggle -> { val dndModeString = getString(DndModeUtils.getLabel(action.dndMode)) getString( - R.string.action_toggle_dnd_mode_formatted, - dndModeString + R.string.action_toggle_dnd_mode_formatted, dndModeString ) } @@ -135,13 +132,11 @@ abstract class BaseActionUiHelper, A : Action>( string = when (action) { is ActionData.Volume.Stream.Decrease -> getString( - R.string.action_decrease_stream_formatted, - streamString + R.string.action_decrease_stream_formatted, streamString ) is ActionData.Volume.Stream.Increase -> getString( - R.string.action_increase_stream_formatted, - streamString + R.string.action_increase_stream_formatted, streamString ) } } @@ -204,8 +199,7 @@ abstract class BaseActionUiHelper, A : Action>( getString(RingerModeUtils.getLabel(action.ringerMode)) string = getString( - R.string.action_change_ringer_mode_formatted, - ringerModeString + R.string.action_change_ringer_mode_formatted, ringerModeString ) } @@ -222,35 +216,31 @@ abstract class BaseActionUiHelper, A : Action>( } } - is ActionData.ControlMediaForApp -> - getAppName(action.packageName).handle( - onSuccess = { appName -> - val resId = when (action) { - is ActionData.ControlMediaForApp.Play -> R.string.action_play_media_package_formatted - is ActionData.ControlMediaForApp.FastForward -> R.string.action_fast_forward_package_formatted - is ActionData.ControlMediaForApp.NextTrack -> R.string.action_next_track_package_formatted - is ActionData.ControlMediaForApp.Pause -> R.string.action_pause_media_package_formatted - is ActionData.ControlMediaForApp.PlayPause -> R.string.action_play_pause_media_package_formatted - is ActionData.ControlMediaForApp.PreviousTrack -> R.string.action_previous_track_package_formatted - is ActionData.ControlMediaForApp.Rewind -> R.string.action_rewind_package_formatted - } + is ActionData.ControlMediaForApp -> getAppName(action.packageName).handle(onSuccess = { appName -> + val resId = when (action) { + is ActionData.ControlMediaForApp.Play -> R.string.action_play_media_package_formatted + is ActionData.ControlMediaForApp.FastForward -> R.string.action_fast_forward_package_formatted + is ActionData.ControlMediaForApp.NextTrack -> R.string.action_next_track_package_formatted + is ActionData.ControlMediaForApp.Pause -> R.string.action_pause_media_package_formatted + is ActionData.ControlMediaForApp.PlayPause -> R.string.action_play_pause_media_package_formatted + is ActionData.ControlMediaForApp.PreviousTrack -> R.string.action_previous_track_package_formatted + is ActionData.ControlMediaForApp.Rewind -> R.string.action_rewind_package_formatted + } - getString(resId, appName) - }, - onError = { - val resId = when (action) { - is ActionData.ControlMediaForApp.Play -> R.string.action_play_media_package - is ActionData.ControlMediaForApp.FastForward -> R.string.action_fast_forward_package - is ActionData.ControlMediaForApp.NextTrack -> R.string.action_next_track_package - is ActionData.ControlMediaForApp.Pause -> R.string.action_pause_media_package - is ActionData.ControlMediaForApp.PlayPause -> R.string.action_play_pause_media_package - is ActionData.ControlMediaForApp.PreviousTrack -> R.string.action_previous_track_package - is ActionData.ControlMediaForApp.Rewind -> R.string.action_rewind_package - } + getString(resId, appName) + }, onError = { + val resId = when (action) { + is ActionData.ControlMediaForApp.Play -> R.string.action_play_media_package + is ActionData.ControlMediaForApp.FastForward -> R.string.action_fast_forward_package + is ActionData.ControlMediaForApp.NextTrack -> R.string.action_next_track_package + is ActionData.ControlMediaForApp.Pause -> R.string.action_pause_media_package + is ActionData.ControlMediaForApp.PlayPause -> R.string.action_play_pause_media_package + is ActionData.ControlMediaForApp.PreviousTrack -> R.string.action_previous_track_package + is ActionData.ControlMediaForApp.Rewind -> R.string.action_rewind_package + } - getString(resId) - } - ) + getString(resId) + }) is ActionData.Flashlight -> { val resId = when (action) { @@ -264,15 +254,17 @@ abstract class BaseActionUiHelper, A : Action>( getString(resId, lensString) } - is ActionData.SwitchKeyboard -> getInputMethodLabel(action.imeId).handle( - onSuccess = { getString(R.string.action_switch_keyboard_formatted, it) }, + is ActionData.SwitchKeyboard -> getInputMethodLabel(action.imeId).handle(onSuccess = { + getString( + R.string.action_switch_keyboard_formatted, + it + ) + }, onError = { getString( - R.string.action_switch_keyboard_formatted, - action.savedImeName + R.string.action_switch_keyboard_formatted, action.savedImeName ) - } - ) + }) is ActionData.Intent -> { val resId = when (action.target) { @@ -288,8 +280,7 @@ abstract class BaseActionUiHelper, A : Action>( is ActionData.TapScreen -> if (action.description.isNullOrBlank()) { getString( - R.string.description_tap_coordinate_default, - arrayOf(action.x, action.y) + R.string.description_tap_coordinate_default, arrayOf(action.x, action.y) ) } else { getString( @@ -300,13 +291,18 @@ abstract class BaseActionUiHelper, A : Action>( is ActionData.SwipeScreen -> if (action.description.isNullOrBlank()) { getString( - R.string.description_swipe_coordinate_default, - arrayOf(action.fingerCount, action.xStart, action.yStart, action.xEnd, action.yEnd, action.duration) + R.string.description_swipe_coordinate_default, arrayOf( + action.fingerCount, + action.xStart, + action.yStart, + action.xEnd, + action.yEnd, + action.duration + ) ) } else { getString( - R.string.description_swipe_coordinate_with_description, - arrayOf( + R.string.description_swipe_coordinate_with_description, arrayOf( action.fingerCount, action.xStart, action.yStart, @@ -318,6 +314,32 @@ abstract class BaseActionUiHelper, A : Action>( ) } + is ActionData.InteractWithScreenElement -> if (action.description.isNullOrBlank()) { + getString( + R.string.description_interact_with_screen_element_default, arrayOf( + getDynamicStringValue( + "extra_label_interact_with_screen_element_interaction_type_${ + action.interactiontype.toString().lowercase() + }" + ), action.elementId, action.appName ?: action.packageName + ) + ) + } else { + getString( + R.string.description_interact_with_screen_element_default_with_description, + arrayOf( + getDynamicStringValue( + "extra_label_interact_with_screen_element_interaction_type_${ + action.interactiontype.toString().lowercase() + }" + ), + action.elementId, + action.appName ?: action.packageName, + action.description + ) + ) + } + is ActionData.Text -> getString(R.string.description_text_block, action.text) is ActionData.Url -> getString(R.string.description_url, action.url) is ActionData.Sound -> getString(R.string.description_sound, action.soundDescription) @@ -380,8 +402,7 @@ abstract class BaseActionUiHelper, A : Action>( } getString( - R.string.action_cycle_rotations_formatted, - orientationStrings.joinToString() + R.string.action_cycle_rotations_formatted, orientationStrings.joinToString() ) } @@ -423,47 +444,41 @@ abstract class BaseActionUiHelper, A : Action>( override fun getIcon(action: ActionData): IconInfo? = when (action) { is ActionData.InputKeyEvent -> null - is ActionData.App -> - getAppIcon(action.packageName).handle( - onSuccess = { IconInfo(it, TintType.None) }, - onError = { null } + is ActionData.App -> getAppIcon(action.packageName).handle(onSuccess = { + IconInfo( + it, + TintType.None ) + }, onError = { null }) is ActionData.AppShortcut -> { if (action.packageName.isNullOrBlank()) { null } else { - getAppIcon(action.packageName).handle( - onSuccess = { IconInfo(it, TintType.None) }, - onError = { null } - ) + getAppIcon(action.packageName).handle(onSuccess = { IconInfo(it, TintType.None) }, + onError = { null }) } } is ActionData.Intent -> null - is ActionData.PhoneCall -> - IconInfo( - getDrawable(R.drawable.ic_outline_call_24), - tintType = TintType.OnSurface - ) + is ActionData.PhoneCall -> IconInfo( + getDrawable(R.drawable.ic_outline_call_24), tintType = TintType.OnSurface + ) is ActionData.TapScreen -> IconInfo( - getDrawable(R.drawable.ic_outline_touch_app_24), - TintType.OnSurface + getDrawable(R.drawable.ic_outline_touch_app_24), TintType.OnSurface ) is ActionData.Text -> null is ActionData.Url -> null is ActionData.Sound -> IconInfo( - getDrawable(R.drawable.ic_outline_volume_up_24), - TintType.OnSurface + getDrawable(R.drawable.ic_outline_volume_up_24), TintType.OnSurface ) else -> ActionUtils.getIcon(action.id)?.let { iconRes -> IconInfo( - getDrawable(iconRes), - TintType.OnSurface + getDrawable(iconRes), TintType.OnSurface ) } } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionViewModel.kt index a5709949cb..94152ba7e8 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionViewModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionViewModel.kt @@ -4,6 +4,7 @@ import android.text.InputType import io.github.sds100.keymapper.R import io.github.sds100.keymapper.actions.swipescreen.SwipePickCoordinateResult import io.github.sds100.keymapper.actions.tapscreen.PickCoordinateResult +import io.github.sds100.keymapper.actions.uielementinteraction.InteractWithScreenElementResult import io.github.sds100.keymapper.system.camera.CameraLens import io.github.sds100.keymapper.system.camera.CameraLensUtils import io.github.sds100.keymapper.system.display.Orientation @@ -25,6 +26,7 @@ import io.github.sds100.keymapper.util.ui.PopupViewModelImpl import io.github.sds100.keymapper.util.ui.ResourceProvider import io.github.sds100.keymapper.util.ui.navigate import io.github.sds100.keymapper.util.ui.showPopup +import timber.log.Timber /** * Created by sds100 on 26/07/2021. @@ -80,25 +82,18 @@ class CreateActionViewModelImpl( val action = when (actionId) { ActionId.PAUSE_MEDIA_PACKAGE -> ActionData.ControlMediaForApp.Pause(packageName) - ActionId.PLAY_MEDIA_PACKAGE -> ActionData.ControlMediaForApp.Play(packageName) - ActionId.PLAY_PAUSE_MEDIA_PACKAGE -> ActionData.ControlMediaForApp.PlayPause(packageName) - ActionId.NEXT_TRACK_PACKAGE -> ActionData.ControlMediaForApp.NextTrack(packageName) - ActionId.PREVIOUS_TRACK_PACKAGE -> ActionData.ControlMediaForApp.PreviousTrack(packageName) - ActionId.FAST_FORWARD_PACKAGE -> ActionData.ControlMediaForApp.FastForward(packageName) - ActionId.REWIND_PACKAGE -> ActionData.ControlMediaForApp.Rewind(packageName) - else -> throw Exception("don't know how to create action for $actionId") } @@ -144,7 +139,6 @@ class CreateActionViewModelImpl( ActionId.VOLUME_TOGGLE_MUTE -> ActionData.Volume.ToggleMute( showVolumeUi ) - else -> throw Exception("don't know how to create action for $actionId") } @@ -371,6 +365,41 @@ class CreateActionViewModelImpl( ) } + ActionId.INTERACT_WITH_SCREEN_ELEMENT -> { + val oldResult = if (oldData is ActionData.InteractWithScreenElement) { + InteractWithScreenElementResult( + oldData.elementId, + oldData.packageName, + oldData.fullName, + oldData.appName, + oldData.onlyIfVisible, + oldData.interactiontype, + oldData.description ?: "" + ) + } else { + null + } + + val result = navigate( + "interact_with_screen_element", + NavDestination.InteractWithScreenElement(oldResult) + ) ?: return null + + val description = result.description.ifEmpty { + null + } + + return ActionData.InteractWithScreenElement( + result.elementId, + result.packageName, + result.fullName, + result.appName, + result.onlyIfVisible, + result.interactionType, + description + ) + } + ActionId.TEXT -> { val oldText = if (oldData is ActionData.Text) { oldData.text diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt b/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt index d25107c329..efe36081f6 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt @@ -43,21 +43,7 @@ import io.github.sds100.keymapper.system.url.OpenUrlAdapter import io.github.sds100.keymapper.system.volume.RingerMode import io.github.sds100.keymapper.system.volume.VolumeAdapter import io.github.sds100.keymapper.system.volume.VolumeStream -import io.github.sds100.keymapper.util.Error -import io.github.sds100.keymapper.util.Event -import io.github.sds100.keymapper.util.InputEventType -import io.github.sds100.keymapper.util.Result -import io.github.sds100.keymapper.util.Success -import io.github.sds100.keymapper.util.dataOrNull -import io.github.sds100.keymapper.util.firstBlocking -import io.github.sds100.keymapper.util.getFullMessage -import io.github.sds100.keymapper.util.getWordBoundaries -import io.github.sds100.keymapper.util.ifIsData -import io.github.sds100.keymapper.util.onFailure -import io.github.sds100.keymapper.util.onSuccess -import io.github.sds100.keymapper.util.otherwise -import io.github.sds100.keymapper.util.success -import io.github.sds100.keymapper.util.then +import io.github.sds100.keymapper.util.* import io.github.sds100.keymapper.util.ui.ResourceProvider import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.delay @@ -127,11 +113,9 @@ class PerformActionsUseCaseImpl( is ActionData.App -> { result = packageManagerAdapter.openApp(action.packageName) } - is ActionData.AppShortcut -> { result = appShortcutAdapter.launchShortcut(action.uri) } - is ActionData.Intent -> { result = intentAdapter.send(action.target, action.uri) } @@ -169,7 +153,6 @@ class PerformActionsUseCaseImpl( is ActionData.DoNotDisturb.Enable -> { result = volumeAdapter.enableDndMode(action.dndMode) } - is ActionData.DoNotDisturb.Toggle -> { result = if (volumeAdapter.isDndEnabled()) { volumeAdapter.disableDndMode() @@ -185,27 +168,21 @@ class PerformActionsUseCaseImpl( is ActionData.ControlMediaForApp.FastForward -> { result = mediaAdapter.fastForward(action.packageName) } - is ActionData.ControlMediaForApp.NextTrack -> { result = mediaAdapter.nextTrack(action.packageName) } - is ActionData.ControlMediaForApp.Pause -> { result = mediaAdapter.pause(action.packageName) } - is ActionData.ControlMediaForApp.Play -> { result = mediaAdapter.play(action.packageName) } - is ActionData.ControlMediaForApp.PlayPause -> { result = mediaAdapter.playPause(action.packageName) } - is ActionData.ControlMediaForApp.PreviousTrack -> { result = mediaAdapter.previousTrack(action.packageName) } - is ActionData.ControlMediaForApp.Rewind -> { result = mediaAdapter.rewind(action.packageName) } @@ -258,7 +235,6 @@ class PerformActionsUseCaseImpl( is ActionData.Volume.Down -> { result = volumeAdapter.lowerVolume(showVolumeUi = action.showVolumeUi) } - is ActionData.Volume.Up -> { result = volumeAdapter.raiseVolume(showVolumeUi = action.showVolumeUi) } @@ -294,15 +270,11 @@ class PerformActionsUseCaseImpl( } is ActionData.SwipeScreen -> { - result = accessibilityService.swipeScreen( - action.xStart, - action.yStart, - action.xEnd, - action.yEnd, - action.fingerCount, - action.duration, - inputEventType - ) + result = accessibilityService.swipeScreen(action.xStart, action.yStart, action.xEnd, action.yEnd, action.fingerCount, action.duration, inputEventType) + } + + is ActionData.InteractWithScreenElement -> { + result = accessibilityService.interactWithScreenElement(action.fullName, action.onlyIfVisible, action.interactiontype, inputEventType) } is ActionData.Text -> { @@ -383,11 +355,9 @@ class PerformActionsUseCaseImpl( is ActionData.Brightness.EnableAuto -> { result = displayAdapter.enableAutoBrightness() } - is ActionData.Brightness.Increase -> { result = displayAdapter.increaseBrightness() } - is ActionData.Brightness.Decrease -> { result = displayAdapter.decreaseBrightness() } @@ -459,7 +429,7 @@ class PerformActionsUseCaseImpl( is ActionData.StatusBar.ToggleNotifications -> { result = if (accessibilityService.rootNode?.packageName == "com.android.systemui") { - closeStatusBarShade() + closeStatusBarShade() } else { val globalAction = AccessibilityService.GLOBAL_ACTION_NOTIFICATIONS @@ -498,27 +468,21 @@ class PerformActionsUseCaseImpl( is ActionData.ControlMedia.Pause -> { result = mediaAdapter.pause() } - is ActionData.ControlMedia.Play -> { result = mediaAdapter.play() } - is ActionData.ControlMedia.PlayPause -> { result = mediaAdapter.playPause() } - is ActionData.ControlMedia.NextTrack -> { result = mediaAdapter.nextTrack() } - is ActionData.ControlMedia.PreviousTrack -> { result = mediaAdapter.previousTrack() } - is ActionData.ControlMedia.FastForward -> { result = mediaAdapter.fastForward() } - is ActionData.ControlMedia.Rewind -> { result = mediaAdapter.rewind() } @@ -527,7 +491,6 @@ class PerformActionsUseCaseImpl( result = accessibilityService.doGlobalAction(AccessibilityService.GLOBAL_ACTION_BACK) } - is ActionData.GoHome -> { result = accessibilityService.doGlobalAction(AccessibilityService.GLOBAL_ACTION_HOME) @@ -564,11 +527,9 @@ class PerformActionsUseCaseImpl( is ActionData.Nfc.Enable -> { result = nfcAdapter.enable() } - is ActionData.Nfc.Disable -> { result = nfcAdapter.disable() } - is ActionData.Nfc.Toggle -> { result = if (nfcAdapter.isEnabled()) { nfcAdapter.disable() @@ -629,7 +590,6 @@ class PerformActionsUseCaseImpl( AccessibilityNodeAction(AccessibilityNodeInfo.ACTION_COPY) } } - is ActionData.PasteText -> { result = accessibilityService.performActionOnNode({ it.isFocused }) { AccessibilityNodeAction(AccessibilityNodeInfo.ACTION_PASTE) @@ -671,11 +631,9 @@ class PerformActionsUseCaseImpl( airplaneModeAdapter.enable() } } - is ActionData.AirplaneMode.Enable -> { result = airplaneModeAdapter.enable() } - is ActionData.AirplaneMode.Disable -> { result = airplaneModeAdapter.disable() } @@ -707,11 +665,9 @@ class PerformActionsUseCaseImpl( is ActionData.VoiceAssistant -> { result = packageManagerAdapter.launchVoiceAssistant() } - is ActionData.DeviceAssistant -> { result = packageManagerAdapter.launchDeviceAssistant() } - is ActionData.OpenCamera -> { result = packageManagerAdapter.launchCameraApp() } @@ -764,13 +720,13 @@ class PerformActionsUseCaseImpl( result = null } - - ActionData.AnswerCall -> { + + ActionData.AnswerCall ->{ phoneAdapter.answerCall() result = success() } - - ActionData.EndCall -> { + + ActionData.EndCall ->{ phoneAdapter.endCall() result = success() } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/swipescreen/SwipePickDisplayCoordinateFragment.kt b/app/src/main/java/io/github/sds100/keymapper/actions/swipescreen/SwipePickDisplayCoordinateFragment.kt index a7eb2006b8..84d278079e 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/swipescreen/SwipePickDisplayCoordinateFragment.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/swipescreen/SwipePickDisplayCoordinateFragment.kt @@ -29,6 +29,7 @@ import kotlinx.coroutines.flow.collectLatest import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json +import timber.log.Timber class SwipePickDisplayCoordinateFragment : Fragment() { companion object { diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/swipescreen/SwipePickDisplayCoordinateViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/actions/swipescreen/SwipePickDisplayCoordinateViewModel.kt index c4a713011a..57a9bb2d6e 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/swipescreen/SwipePickDisplayCoordinateViewModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/swipescreen/SwipePickDisplayCoordinateViewModel.kt @@ -9,19 +9,19 @@ import io.github.sds100.keymapper.R import io.github.sds100.keymapper.util.ui.* import kotlinx.coroutines.flow.* import kotlinx.coroutines.launch +import timber.log.Timber import kotlin.math.roundToInt enum class ScreenshotTouchType { START, END } - class SwipePickDisplayCoordinateViewModel( resourceProvider: ResourceProvider ) : ViewModel(), ResourceProvider by resourceProvider, PopupViewModel by PopupViewModelImpl() { - val screenshotTouchTypeStart = ScreenshotTouchType.START - val screenshotTouchTypeEnd = ScreenshotTouchType.END + public val screenshotTouchTypeStart = ScreenshotTouchType.START; + public val screenshotTouchTypeEnd = ScreenshotTouchType.END; private val xStart = MutableStateFlow(null) private val yStart = MutableStateFlow(null) private val xEnd = MutableStateFlow(null) @@ -74,19 +74,18 @@ class SwipePickDisplayCoordinateViewModel( val bitmap = _bitmap.asStateFlow() val returnResult = _returnResult.asSharedFlow() - val isSelectStartEndSwitchEnabled: StateFlow = combine(bitmap) { - bitmap.value != null + val isSelectStartEndSwitchEnabled:StateFlow = combine(bitmap) { + bitmap?.value != null }.stateIn(viewModelScope, SharingStarted.Lazily, false) - private val isCoordinatesValid: StateFlow = - combine(xStart, yStart, xEnd, yEnd) { xStart, yStart, xEnd, yEnd -> - xStart ?: return@combine false - yStart ?: return@combine false - xEnd ?: return@combine false - yEnd ?: return@combine false + private val isCoordinatesValid: StateFlow = combine(xStart, yStart, xEnd, yEnd) { xStart, yStart, xEnd, yEnd -> + xStart ?: return@combine false + yStart ?: return@combine false + xEnd ?: return@combine false + yEnd ?: return@combine false - xStart >= 0 && yStart >= 0 && xEnd >= 0 && yEnd >= 0 - }.stateIn(viewModelScope, SharingStarted.Lazily, false) + xStart >= 0 && yStart >= 0 && xEnd >= 0 && yEnd >= 0 + }.stateIn(viewModelScope, SharingStarted.Lazily, false) private val isOptionsValid: StateFlow = combine(fingerCount, duration) { fingerCount, duration -> fingerCount ?: return@combine false @@ -95,14 +94,13 @@ class SwipePickDisplayCoordinateViewModel( fingerCount >= 1 && duration > 0 }.stateIn(viewModelScope, SharingStarted.Lazily, false) - val isDoneButtonEnabled: StateFlow = - combine(isCoordinatesValid, isOptionsValid) { isCoordinatesValid, isOptionsValid -> - isCoordinatesValid && isOptionsValid - }.stateIn(viewModelScope, SharingStarted.Lazily, false) + val isDoneButtonEnabled: StateFlow = combine(isCoordinatesValid, isOptionsValid) { isCoordinatesValid, isOptionsValid -> + isCoordinatesValid && isOptionsValid + }.stateIn(viewModelScope, SharingStarted.Lazily, false) fun selectedScreenshot(newBitmap: Bitmap, displaySize: Point) { - _screenshotTouchType.value = ScreenshotTouchType.START + _screenshotTouchType.value = ScreenshotTouchType.START; //check whether the height and width of the bitmap match the display size, even when it is rotated. if ( @@ -150,7 +148,7 @@ class SwipePickDisplayCoordinateViewModel( this.duration.value = duration.toIntOrNull() } - fun setStartOrEndCoordinates(isChecked: Boolean, type: ScreenshotTouchType) { + fun setStartOrEndCoordinates(isChecked:Boolean, type: ScreenshotTouchType) { if (isChecked) this._screenshotTouchType.value = type } @@ -192,17 +190,7 @@ class SwipePickDisplayCoordinateViewModel( ) ) ?: return@launch - _returnResult.emit( - SwipePickCoordinateResult( - xStart, - yStart, - xEnd, - yEnd, - fingerCount, - duration, - description - ) - ) + _returnResult.emit(SwipePickCoordinateResult(xStart, yStart, xEnd, yEnd, fingerCount, duration, description)) } } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielementinteraction/InteractWithScreenElementFragment.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielementinteraction/InteractWithScreenElementFragment.kt new file mode 100644 index 0000000000..b632672a8d --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielementinteraction/InteractWithScreenElementFragment.kt @@ -0,0 +1,118 @@ +package io.github.sds100.keymapper.actions.uielementinteraction + +import android.annotation.SuppressLint +import android.os.Bundle +import android.view.LayoutInflater +import android.view.View +import android.view.ViewGroup +import android.widget.ArrayAdapter +import androidx.activity.addCallback +import androidx.core.os.bundleOf +import androidx.fragment.app.Fragment +import androidx.fragment.app.setFragmentResult +import androidx.fragment.app.viewModels +import androidx.lifecycle.Lifecycle +import androidx.navigation.fragment.findNavController +import androidx.navigation.fragment.navArgs +import io.github.sds100.keymapper.databinding.FragmentInteractWithScreenElementBinding +import io.github.sds100.keymapper.util.Inject +import io.github.sds100.keymapper.util.getDynamicStringValue +import io.github.sds100.keymapper.util.launchRepeatOnLifecycle +import io.github.sds100.keymapper.util.ui.setupNavigation +import io.github.sds100.keymapper.util.ui.showPopups +import kotlinx.coroutines.flow.collectLatest +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import timber.log.Timber +import java.util.Locale + +class InteractWithScreenElementFragment : Fragment() { + companion object { + const val EXTRA_RESULT = "extra_result" + } + + private val args: InteractWithScreenElementFragmentArgs by navArgs() + private val requestKey: String by lazy { args.requestKey } + private var interactionTypesDisplayValues = mutableListOf() + + private val viewModel: InteractWithScreenElementViewModel by viewModels { + Inject.interactWithScreenElementActionTypeViewModel(requireContext()) + } + + private var _binding: FragmentInteractWithScreenElementBinding? = null + val binding: FragmentInteractWithScreenElementBinding + get() = _binding!! + + override fun onCreateView( + inflater: LayoutInflater, + container: ViewGroup?, + savedInstanceState: Bundle? + ): View { + FragmentInteractWithScreenElementBinding.inflate(inflater, container, false).apply { + lifecycleOwner = viewLifecycleOwner + _binding = this + return this.root + } + } + + @SuppressLint("DiscouragedApi") + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + viewModel.setupNavigation(this) + + args.result?.let { + viewModel.loadResult(Json.decodeFromString(it)) + } + + interactionTypesDisplayValues = INTERACTIONTYPE.values().map { + Timber.d("INTERACTION TYPE %s", it.name.lowercase( + Locale.ROOT + )) + val stringName = "extra_label_interact_with_screen_element_interaction_type_${ + it.name.lowercase( + Locale.ROOT + ) + }" + + getDynamicStringValue(stringName) + }.toMutableList() + } + + override fun onViewCreated(view: View, savedInstanceState: Bundle?) { + super.onViewCreated(view, savedInstanceState) + + binding.viewModel = viewModel + binding.interactionTypeSpinnerAdapter = ArrayAdapter( + this.requireActivity(), + android.R.layout.simple_spinner_dropdown_item, + interactionTypesDisplayValues + ) + + viewModel.showPopups(this, binding) + + requireActivity().onBackPressedDispatcher.addCallback(viewLifecycleOwner) { + findNavController().navigateUp() + } + + binding.appBar.setNavigationOnClickListener { + findNavController().navigateUp() + } + + viewLifecycleOwner.launchRepeatOnLifecycle(Lifecycle.State.RESUMED) { + viewModel.returnResult.collectLatest { result -> + setFragmentResult( + requestKey, + bundleOf(EXTRA_RESULT to Json.encodeToString(result)) + ) + findNavController().navigateUp() + } + } + } + + override fun onDestroyView() { + _binding = null + super.onDestroyView() + } +} \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielementinteraction/InteractWithScreenElementResult.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielementinteraction/InteractWithScreenElementResult.kt new file mode 100644 index 0000000000..c62c2ae614 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielementinteraction/InteractWithScreenElementResult.kt @@ -0,0 +1,27 @@ +package io.github.sds100.keymapper.actions.uielementinteraction + +import kotlinx.serialization.Serializable + +enum class INTERACTIONTYPE { + CLICK, + LONG_CLICK, + SELECT, + FOCUS, + CLEAR_FOCUS, + COLLAPSE, + EXPAND, + DISMISS, + SCROLL_FORWARD, + SCROLL_BACKWARD, +} + +@Serializable +data class InteractWithScreenElementResult( + val elementId: String, + val packageName: String, + val fullName: String, + val appName: String?, + val onlyIfVisible: Boolean, + val interactionType: INTERACTIONTYPE, + val description: String +) diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielementinteraction/InteractWithScreenElementViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielementinteraction/InteractWithScreenElementViewModel.kt new file mode 100644 index 0000000000..462015ee2d --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielementinteraction/InteractWithScreenElementViewModel.kt @@ -0,0 +1,161 @@ +package io.github.sds100.keymapper.actions.uielementinteraction + +import android.graphics.drawable.Drawable +import android.view.View +import android.widget.AdapterView +import androidx.lifecycle.ViewModel +import androidx.lifecycle.ViewModelProvider +import androidx.lifecycle.viewModelScope +import io.github.sds100.keymapper.R +import io.github.sds100.keymapper.system.apps.DisplayAppsUseCase +import io.github.sds100.keymapper.util.ui.NavDestination +import io.github.sds100.keymapper.util.ui.NavigationViewModel +import io.github.sds100.keymapper.util.ui.NavigationViewModelImpl +import io.github.sds100.keymapper.util.ui.PopupUi +import io.github.sds100.keymapper.util.ui.PopupViewModel +import io.github.sds100.keymapper.util.ui.PopupViewModelImpl +import io.github.sds100.keymapper.util.ui.ResourceProvider +import io.github.sds100.keymapper.util.ui.navigate +import io.github.sds100.keymapper.util.ui.showPopup +import io.github.sds100.keymapper.util.valueOrNull +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.launch +import java.util.Locale + +class InteractWithScreenElementViewModel( + resourceProvider: ResourceProvider, + displayAppsUseCase: DisplayAppsUseCase, +) : ViewModel(), + ResourceProvider by resourceProvider, + PopupViewModel by PopupViewModelImpl(), + NavigationViewModel by NavigationViewModelImpl(), + DisplayAppsUseCase by displayAppsUseCase { + + private val _interactionTypes = INTERACTIONTYPE.values().map { it.name } + private val _interactionType: MutableStateFlow = + MutableStateFlow(INTERACTIONTYPE.values().first()) + + private val _returnResult = MutableSharedFlow() + val returnResult = _returnResult.asSharedFlow() + + val elementId = MutableStateFlow(null) + val packageName = MutableStateFlow(null) + val fullName = MutableStateFlow(null) + val appName = MutableStateFlow(null) + var appIcon: MutableStateFlow = MutableStateFlow(null) + var onlyIfVisible: MutableStateFlow = MutableStateFlow(true) + + val description: MutableStateFlow = MutableStateFlow(null) + + val interactionTypeSpinnerSelection = _interactionType.map { + it ?: return@map 0 + + this._interactionTypes.indexOf(it.name) + }.stateIn(viewModelScope, SharingStarted.Lazily, 0) + + private fun setInteractionType(type: String) { + _interactionType.value = INTERACTIONTYPE.valueOf(type.uppercase(Locale.ROOT)) + } + + fun onInteractionTypeSelected(parent: AdapterView<*>?, view: View?, position: Int, id: Long) { + this.setInteractionType(_interactionTypes[position]) + } + + fun onChooseUiElementButtonClick() { + viewModelScope.launch { + onSelectUiElement() + } + } + + fun onOnlyIfVisibleCheckboxChange(checked: Boolean) { + onlyIfVisible.value = checked + } + + fun loadResult(result: InteractWithScreenElementResult) { + viewModelScope.launch { + elementId.value = result.elementId + packageName.value = result.packageName + fullName.value = result.fullName + appName.value = result.appName + _interactionType.value = result.interactionType + onlyIfVisible.value = result.onlyIfVisible + description.value = result.description + appIcon.value = getAppIcon(result.packageName).valueOrNull() + } + } + + private suspend fun onSelectUiElement() { + val uiElementInfo = + navigate(NavDestination.ID_CHOOSE_UI_ELEMENT, NavDestination.ChooseUiElement) ?: return + elementId.value = uiElementInfo.elementName + packageName.value = uiElementInfo.packageName + fullName.value = uiElementInfo.fullName + appName.value = uiElementInfo.appName + appIcon.value = getAppIcon(uiElementInfo.packageName).valueOrNull() + } + + val isDoneButtonEnabled: StateFlow = + combine( + elementId, + packageName, + fullName, + appName + ) { elementId, packageName, fullName, appName -> + elementId ?: return@combine false + packageName ?: return@combine false + fullName ?: return@combine false + appName ?: return@combine false + + elementId.isNotEmpty() && packageName.isNotEmpty() && fullName.isNotEmpty() && appName.isNotEmpty() + }.stateIn(viewModelScope, SharingStarted.Lazily, false) + + fun onDoneClick() { + viewModelScope.launch { + val elementId = elementId.value ?: return@launch + val packageName = packageName.value ?: return@launch + val fullName = fullName.value ?: return@launch + val appName = appName.value ?: return@launch + val onlyIfVisible = onlyIfVisible.value ?: return@launch + val interactiontype = _interactionType.value ?: return@launch + + val description = showPopup( + "ui_element_description", + PopupUi.Text( + getString(R.string.hint_interact_with_screen_element_description), + allowEmpty = true, + text = description.value ?: "" + ) + ) ?: return@launch + + _returnResult.emit( + InteractWithScreenElementResult( + elementId, + packageName, + fullName, + appName, + onlyIfVisible, + interactiontype, + description + ) + ) + } + } + + @Suppress("UNCHECKED_CAST") + class Factory( + private val resourceProvider: ResourceProvider, + private val displayAppsUseCase: DisplayAppsUseCase + ) : ViewModelProvider.NewInstanceFactory() { + + override fun create(modelClass: Class): T { + return InteractWithScreenElementViewModel(resourceProvider, displayAppsUseCase) as T + } + } +} \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/data/db/AppDatabase.kt b/app/src/main/java/io/github/sds100/keymapper/data/db/AppDatabase.kt index c1e4be2f05..0b48aef79e 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/db/AppDatabase.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/db/AppDatabase.kt @@ -11,29 +11,22 @@ import io.github.sds100.keymapper.data.db.AppDatabase.Companion.DATABASE_VERSION import io.github.sds100.keymapper.data.db.dao.FingerprintMapDao import io.github.sds100.keymapper.data.db.dao.KeyMapDao import io.github.sds100.keymapper.data.db.dao.LogEntryDao +import io.github.sds100.keymapper.data.db.dao.ViewIdDao import io.github.sds100.keymapper.data.db.typeconverter.ActionListTypeConverter import io.github.sds100.keymapper.data.db.typeconverter.ConstraintListTypeConverter import io.github.sds100.keymapper.data.db.typeconverter.ExtraListTypeConverter import io.github.sds100.keymapper.data.db.typeconverter.TriggerTypeConverter +import io.github.sds100.keymapper.data.entities.LogEntryEntity +import io.github.sds100.keymapper.data.migration.* import io.github.sds100.keymapper.data.entities.FingerprintMapEntity import io.github.sds100.keymapper.data.entities.KeyMapEntity -import io.github.sds100.keymapper.data.entities.LogEntryEntity -import io.github.sds100.keymapper.data.migration.Migration_10_11 -import io.github.sds100.keymapper.data.migration.Migration_11_12 -import io.github.sds100.keymapper.data.migration.Migration_1_2 -import io.github.sds100.keymapper.data.migration.Migration_2_3 -import io.github.sds100.keymapper.data.migration.Migration_3_4 -import io.github.sds100.keymapper.data.migration.Migration_4_5 -import io.github.sds100.keymapper.data.migration.Migration_5_6 -import io.github.sds100.keymapper.data.migration.Migration_6_7 -import io.github.sds100.keymapper.data.migration.Migration_8_9 -import io.github.sds100.keymapper.data.migration.Migration_9_10 +import io.github.sds100.keymapper.data.entities.ViewIdEntity /** * Created by sds100 on 24/01/2020. */ @Database( - entities = [KeyMapEntity::class, FingerprintMapEntity::class, LogEntryEntity::class], + entities = [KeyMapEntity::class, FingerprintMapEntity::class, LogEntryEntity::class, ViewIdEntity::class], version = DATABASE_VERSION, exportSchema = true ) @@ -46,7 +39,7 @@ import io.github.sds100.keymapper.data.migration.Migration_9_10 abstract class AppDatabase : RoomDatabase() { companion object { const val DATABASE_NAME = "key_map_database" - const val DATABASE_VERSION = 13 + const val DATABASE_VERSION = 14 val MIGRATION_1_2 = object : Migration(1, 2) { @@ -115,6 +108,12 @@ abstract class AppDatabase : RoomDatabase() { database.execSQL("CREATE TABLE `log` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `time` INTEGER NOT NULL, `severity` INTEGER NOT NULL, `message` TEXT NOT NULL)") } } + + val MIGRATION_13_14 = object: Migration(13, 14) { + override fun migrate(database: SupportSQLiteDatabase) { + database.execSQL("CREATE TABLE `viewids` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `view_id` VARCHAR(250) NOT NULL, `package_name` VARCHAR(250) NOT NULL), `full_name` TEXT NOT NULL") + } + } } class RoomMigration_11_12( @@ -128,4 +127,6 @@ abstract class AppDatabase : RoomDatabase() { abstract fun keymapDao(): KeyMapDao abstract fun fingerprintMapDao(): FingerprintMapDao abstract fun logEntryDao(): LogEntryDao + + abstract fun viewIdDao(): ViewIdDao } \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/data/db/dao/ViewIdDao.kt b/app/src/main/java/io/github/sds100/keymapper/data/db/dao/ViewIdDao.kt new file mode 100644 index 0000000000..5a9b391e00 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/data/db/dao/ViewIdDao.kt @@ -0,0 +1,34 @@ +package io.github.sds100.keymapper.data.db.dao + +import androidx.room.Dao +import androidx.room.Insert +import androidx.room.OnConflictStrategy +import androidx.room.Query +import io.github.sds100.keymapper.data.entities.ViewIdEntity +import kotlinx.coroutines.flow.Flow + +@Dao +interface ViewIdDao { + companion object { + const val TABLE_NAME = "viewids" + const val KEY_ID = "id" + const val KEY_ELEMENT_ID = "view_id" + const val KEY_PACKAGE_NAME = "package_name" + const val KEY_FULL_NAME = "full_name" + } + + @Query("SELECT * FROM $TABLE_NAME ORDER BY $KEY_PACKAGE_NAME ASC, $KEY_ELEMENT_ID ASC") + fun getAll(): Flow> + + @Query("SELECT * FROM $TABLE_NAME WHERE $KEY_ID = :id") + fun getById(id: Long): ViewIdEntity + + @Query("SELECT EXISTS(SELECT * FROM $TABLE_NAME WHERE $KEY_FULL_NAME = :fullName)") + fun viewIdExists(fullName: String): Boolean + + @Query("DELETE FROM $TABLE_NAME") + suspend fun deleteAll() + + @Insert(onConflict = OnConflictStrategy.ABORT) + suspend fun insert(vararg entry: ViewIdEntity) +} \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt b/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt index 2a7591b2ca..da013ede47 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt @@ -75,6 +75,7 @@ data class ActionEntity( const val EXTRA_DND_MODE = "extra_do_not_disturb_mode" const val EXTRA_ORIENTATIONS = "extra_orientations" const val EXTRA_COORDINATE_DESCRIPTION = "extra_coordinate_description" + const val EXTRA_ELEMENT_DESCRIPTION = "extra_element_description" const val EXTRA_INTENT_TARGET = "extra_intent_target" const val EXTRA_INTENT_DESCRIPTION = "extra_intent_description" const val EXTRA_SOUND_FILE_DESCRIPTION = "extra_sound_file_description" @@ -126,7 +127,7 @@ data class ActionEntity( enum class Type { //DONT CHANGE THESE - APP, APP_SHORTCUT, KEY_EVENT, TEXT_BLOCK, URL, SYSTEM_ACTION, TAP_COORDINATE, SWIPE_COORDINATE, INTENT, PHONE_CALL, SOUND + APP, APP_SHORTCUT, KEY_EVENT, TEXT_BLOCK, URL, SYSTEM_ACTION, TAP_COORDINATE, SWIPE_COORDINATE, INTERACT_WITH_SCREEN_ELEMENT, INTENT, PHONE_CALL, SOUND } constructor( diff --git a/app/src/main/java/io/github/sds100/keymapper/data/entities/ViewIdEntity.kt b/app/src/main/java/io/github/sds100/keymapper/data/entities/ViewIdEntity.kt new file mode 100644 index 0000000000..34f037d64e --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/data/entities/ViewIdEntity.kt @@ -0,0 +1,25 @@ +package io.github.sds100.keymapper.data.entities + +import androidx.room.ColumnInfo +import androidx.room.Entity +import androidx.room.PrimaryKey +import io.github.sds100.keymapper.data.db.dao.ViewIdDao + +@Entity(tableName = ViewIdDao.TABLE_NAME) +data class ViewIdEntity( + @PrimaryKey(autoGenerate = true) + val id: Int, + @ColumnInfo(name = ViewIdDao.KEY_ELEMENT_ID) + val viewId: String, + @ColumnInfo(name = ViewIdDao.KEY_PACKAGE_NAME) + val packageName: String, + @ColumnInfo(name = ViewIdDao.KEY_FULL_NAME,) + val fullName: String, +){ + companion object{ + const val NAME_ID = "id" + const val NAME_VIEW_ID = "viewId" + const val NAME_PACKAGE_NAME = "packageName" + const val NAME_FULL_NAME = "fullName" + } +} \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/data/repositories/RoomViewIdRepository.kt b/app/src/main/java/io/github/sds100/keymapper/data/repositories/RoomViewIdRepository.kt new file mode 100644 index 0000000000..12209a567d --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/data/repositories/RoomViewIdRepository.kt @@ -0,0 +1,61 @@ +package io.github.sds100.keymapper.data.repositories + +import io.github.sds100.keymapper.data.db.dao.ViewIdDao +import io.github.sds100.keymapper.data.entities.ViewIdEntity +import io.github.sds100.keymapper.util.DefaultDispatcherProvider +import io.github.sds100.keymapper.util.DispatcherProvider +import io.github.sds100.keymapper.util.State +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.launch + +class RoomViewIdRepository( + private val dao: ViewIdDao, + private val coroutineScope: CoroutineScope, + dispatchers: DispatcherProvider = DefaultDispatcherProvider() +): ViewIdRepository { + + override val viewIdList = dao.getAll() + .map { State.Data(it) } + .flowOn(dispatchers.default()) + .stateIn(coroutineScope, SharingStarted.Eagerly, State.Loading) + + init { + // clean up the DB on every app start + /*coroutineScope.launch(Dispatchers.Default) { + dao.deleteAll() + }*/ + } + + override fun insert(entry: ViewIdEntity) { + coroutineScope.launch(Dispatchers.Default) { + if (!dao.viewIdExists(entry.fullName)) { + dao.insert(entry) + } + } + } + + override fun deleteAll() { + coroutineScope.launch(Dispatchers.Default) { + dao.deleteAll() + } + } + + override suspend fun getAll() { + coroutineScope.launch(Dispatchers.Default) { + dao.getAll() + } + } +} + +interface ViewIdRepository { + val viewIdList: Flow>> + fun insert(entry: ViewIdEntity) + fun deleteAll() + suspend fun getAll() +} \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityServiceController.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityServiceController.kt index 33a092e867..fcdbe50a01 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityServiceController.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityServiceController.kt @@ -1,16 +1,20 @@ package io.github.sds100.keymapper.system.accessibility import android.accessibilityservice.AccessibilityServiceInfo +import android.annotation.SuppressLint import android.os.Build import android.os.SystemClock import android.view.KeyEvent import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo +import io.github.sds100.keymapper.BuildConfig import io.github.sds100.keymapper.actions.ActionData import io.github.sds100.keymapper.actions.PerformActionsUseCase import io.github.sds100.keymapper.constraints.DetectConstraintsUseCase import io.github.sds100.keymapper.data.Keys +import io.github.sds100.keymapper.data.entities.ViewIdEntity import io.github.sds100.keymapper.data.repositories.PreferenceRepository +import io.github.sds100.keymapper.data.repositories.ViewIdRepository import io.github.sds100.keymapper.mappings.PauseMappingsUseCase import io.github.sds100.keymapper.mappings.fingerprintmaps.DetectFingerprintMapsUseCase import io.github.sds100.keymapper.mappings.fingerprintmaps.FingerprintGestureMapController @@ -21,6 +25,8 @@ import io.github.sds100.keymapper.mappings.keymaps.detection.DetectScreenOffKeyE import io.github.sds100.keymapper.mappings.keymaps.detection.KeyMapController import io.github.sds100.keymapper.reroutekeyevents.RerouteKeyEventsController import io.github.sds100.keymapper.reroutekeyevents.RerouteKeyEventsUseCase +import io.github.sds100.keymapper.system.apps.PACKAGE_INFO_TYPES +import io.github.sds100.keymapper.system.apps.PackageUtils import io.github.sds100.keymapper.system.devices.DevicesAdapter import io.github.sds100.keymapper.system.devices.InputDeviceInfo import io.github.sds100.keymapper.system.inputmethod.InputMethodAdapter @@ -68,7 +74,8 @@ class AccessibilityServiceController( private val devicesAdapter: DevicesAdapter, private val suAdapter: SuAdapter, private val inputMethodAdapter: InputMethodAdapter, - private val settingsRepository: PreferenceRepository + private val settingsRepository: PreferenceRepository, + private val viewIdRepository: ViewIdRepository ) { companion object { @@ -76,6 +83,19 @@ class AccessibilityServiceController( * How long should the accessibility service record a trigger in seconds. */ private const val RECORD_TRIGGER_TIMER_LENGTH = 5 + + /** + * How long should the accessibility service record UI elements in seconds + */ + private const val RECORD_UI_ELEMENTS_TIMER_LENGTH = 5 * 60 + /** + * On which events we want to record UI Elements? + */ + private val RECORD_UI_ELEMENTS_EVENT_TYPES = intArrayOf( + //AccessibilityEvent.TYPE_WINDOWS_CHANGED, + //AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED, + AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED + ) } private val triggerKeyMapFromOtherAppsController = TriggerKeyMapFromOtherAppsController( @@ -133,10 +153,15 @@ class AccessibilityServiceController( } } + private var recordingUiElementsJob: Job? = null + private val recordingUiElements: Boolean + get() = recordingUiElementsJob != null && recordingUiElementsJob?.isActive == true + private val initialServiceFlags: Int by lazy { var flags = AccessibilityServiceInfo.FLAG_REQUEST_FILTER_KEY_EVENTS .withFlag(AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS) .withFlag(AccessibilityServiceInfo.DEFAULT) + .withFlag(AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS) .withFlag(AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { @@ -146,6 +171,16 @@ class AccessibilityServiceController( return@lazy flags } + private val initialFeedbackFlags: Int by lazy { + + return@lazy AccessibilityServiceInfo.FEEDBACK_ALL_MASK + } + + private val initialEventTypes: Int by lazy { + + return@lazy AccessibilityEvent.TYPE_WINDOWS_CHANGED.withFlag(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED).withFlag(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) + } + /* On some devices the onServiceConnected method is called multiple times throughout the lifecycle of the service. The service flags that the controller *expects* will be stored here. Whenever onServiceConnected is called the @@ -154,8 +189,8 @@ class AccessibilityServiceController( */ private var serviceFlags: MutableStateFlow = MutableStateFlow(initialServiceFlags) - private var serviceFeedbackType: MutableStateFlow = MutableStateFlow(0) - private var serviceEventTypes: MutableStateFlow = MutableStateFlow(0) + private var serviceFeedbackType: MutableStateFlow = MutableStateFlow(initialFeedbackFlags) + private var serviceEventTypes: MutableStateFlow = MutableStateFlow(initialEventTypes) init { serviceFlags.onEach { flags -> @@ -407,8 +442,38 @@ class AccessibilityServiceController( ) } - fun onAccessibilityEvent(event: AccessibilityEventModel?) { + fun onAccessibilityEvent(event: AccessibilityEventModel?, originalEvent: AccessibilityEvent?) { Timber.d("OnAccessibilityEvent $event") + + // TODO: DECREASE EVENTS!!! + + /** + * Record UI elements and store them into the DB + */ + if (recordingUiElements && originalEvent != null && RECORD_UI_ELEMENTS_EVENT_TYPES.contains(originalEvent.eventType)) { + val foundViewIds = accessibilityService.fetchAvailableUIElements() + + if (foundViewIds.isNotEmpty()) { + foundViewIds.forEachIndexed { _, item -> + val elementId = PackageUtils.getInfoFromFullyQualifiedViewName(item, PACKAGE_INFO_TYPES.TYPE_VIEW_ID) + val packageName = PackageUtils.getInfoFromFullyQualifiedViewName(item, PACKAGE_INFO_TYPES.TYPE_PACKAGE_NAME) + + if (elementId != null && packageName != null && packageName != BuildConfig.APPLICATION_ID) { + viewIdRepository.insert( + ViewIdEntity( + id = 0, + viewId = elementId, + packageName = packageName, + fullName = item + ) + ) + } + } + } + + } + + val focussedNode = accessibilityService.findFocussedNode(AccessibilityNodeInfo.FOCUS_INPUT) if (focussedNode?.isEditable == true && focussedNode.isFocused) { @@ -432,7 +497,8 @@ class AccessibilityServiceController( triggerKeyMapFromOtherAppsController.onDetected(uid) } - private fun onEventFromUi(event: Event) { + @SuppressLint("NewApi") + private suspend fun onEventFromUi(event: Event) { Timber.d("Service received event from UI: $event") when (event) { is Event.StartRecordingTrigger -> @@ -454,12 +520,30 @@ class AccessibilityServiceController( } is Event.TestAction -> performActionsUseCase.perform(event.action) - is Event.Ping -> coroutineScope.launch { outputEvents.emit(Event.Pong(event.key)) } is Event.HideKeyboard -> accessibilityService.hideKeyboard() is Event.ShowKeyboard -> accessibilityService.showKeyboard() is Event.ChangeIme -> accessibilityService.switchIme(event.imeId) is Event.DisableService -> accessibilityService.disableSelf() + is Event.StartRecordingUiElements -> + if (!recordingUiElements) { + recordingUiElementsJob = recordUiElementsJob() + } + is Event.StopRecordingUiElements -> { + val wasRecordingUiElements = recordingUiElements + + recordingUiElementsJob?.cancel() + recordingUiElementsJob = null + + if (wasRecordingUiElements) { + coroutineScope.launch { + outputEvents.emit(Event.OnStoppedRecordingUiElements) + } + } + } + is Event.ClearRecordedUiElements -> coroutineScope.launch { + viewIdRepository.deleteAll() + } else -> Unit } } @@ -477,6 +561,19 @@ class AccessibilityServiceController( outputEvents.emit(Event.OnStoppedRecordingTrigger) } + private fun recordUiElementsJob() = coroutineScope.launch { + repeat(RECORD_UI_ELEMENTS_TIMER_LENGTH) {iteration -> + if (isActive) { + val timeLeft = RECORD_UI_ELEMENTS_TIMER_LENGTH - iteration + outputEvents.emit(Event.OnIncrementRecordUiElementsTimer(timeLeft)) + + delay(1000) + } + } + + outputEvents.emit(Event.OnStoppedRecordingUiElements) + } + private fun requestFingerprintGestureDetection() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { Timber.d("Accessibility service: request fingerprint gesture detection") diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/IAccessibilityService.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/IAccessibilityService.kt index 44e7b20d4d..478bab8825 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/IAccessibilityService.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/IAccessibilityService.kt @@ -2,6 +2,7 @@ package io.github.sds100.keymapper.system.accessibility import android.os.Build import androidx.annotation.RequiresApi +import io.github.sds100.keymapper.actions.uielementinteraction.INTERACTIONTYPE import io.github.sds100.keymapper.util.InputEventType import io.github.sds100.keymapper.util.Result import kotlinx.coroutines.flow.Flow @@ -13,22 +14,17 @@ interface IAccessibilityService { fun doGlobalAction(action: Int): Result<*> fun tapScreen(x: Int, y: Int, inputEventType: InputEventType): Result<*> - fun swipeScreen( - xStart: Int, - yStart: Int, - xEnd: Int, - yEnd: Int, - fingerCount: Int, - duration: Int, - inputEventType: InputEventType - ): Result<*> + fun swipeScreen(xStart: Int, yStart: Int, xEnd: Int, yEnd: Int, fingerCount: Int, duration: Int, inputEventType: InputEventType): Result<*> + fun interactWithScreenElement(fullName: String, onlyIfVisible: Boolean, interactiontype: INTERACTIONTYPE, inputEventType: InputEventType): Result<*> + + fun fetchAvailableUIElements(): List val isFingerprintGestureDetectionAvailable: Boolean var serviceFlags: Int? var serviceFeedbackType: Int? var serviceEventTypes: Int? - + fun performActionOnNode( findNode: (node: AccessibilityNodeModel) -> Boolean, performAction: (node: AccessibilityNodeModel) -> AccessibilityNodeAction? diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt index 312959306a..7397bb7b88 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt @@ -18,11 +18,13 @@ import android.os.Build import android.os.IBinder import android.view.KeyEvent import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo import androidx.core.content.getSystemService import androidx.core.os.bundleOf import androidx.lifecycle.Lifecycle import androidx.lifecycle.LifecycleOwner import androidx.lifecycle.LifecycleRegistry +import io.github.sds100.keymapper.actions.uielementinteraction.INTERACTIONTYPE import io.github.sds100.keymapper.api.Api import io.github.sds100.keymapper.api.IKeyEventReceiver import io.github.sds100.keymapper.api.IKeyEventReceiverCallback @@ -68,7 +70,7 @@ class MyAccessibilityService : AccessibilityService(), LifecycleOwner, IAccessib private lateinit var lifecycleRegistry: LifecycleRegistry private var fingerprintGestureCallback: - FingerprintGestureController.FingerprintGestureCallback? = null + FingerprintGestureController.FingerprintGestureCallback? = null override val rootNode: AccessibilityNodeModel? get() { @@ -282,7 +284,7 @@ class MyAccessibilityService : AccessibilityService(), LifecycleOwner, IAccessib } override fun onAccessibilityEvent(event: AccessibilityEvent?) { - controller?.onAccessibilityEvent(event?.toModel()) + controller?.onAccessibilityEvent(event?.toModel(), event) } override fun onKeyEvent(event: KeyEvent?): Boolean { @@ -296,12 +298,12 @@ class MyAccessibilityService : AccessibilityService(), LifecycleOwner, IAccessib if (controller != null) { return controller!!.onKeyEvent( - event.keyCode, - event.action, - device, - event.metaState, - event.scanCode, - event.eventTime + event.keyCode, + event.action, + device, + event.metaState, + event.scanCode, + event.eventTime ) } @@ -330,8 +332,8 @@ class MyAccessibilityService : AccessibilityService(), LifecycleOwner, IAccessib } override fun performActionOnNode( - findNode: (node: AccessibilityNodeModel) -> Boolean, - performAction: (node: AccessibilityNodeModel) -> AccessibilityNodeAction? + findNode: (node: AccessibilityNodeModel) -> Boolean, + performAction: (node: AccessibilityNodeModel) -> AccessibilityNodeAction? ): Result<*> { val node = rootInActiveWindow.findNodeRecursively { findNode(it.toModel()) @@ -352,10 +354,10 @@ class MyAccessibilityService : AccessibilityService(), LifecycleOwner, IAccessib override fun doGlobalAction(action: Int): Result<*> { val success = performGlobalAction(action) - if (success) { - return Success(Unit) + return if (success) { + Success(Unit) } else { - return Error.FailedToPerformAccessibilityGlobalAction(action) + Error.FailedToPerformAccessibilityGlobalAction(action) } } @@ -369,25 +371,25 @@ class MyAccessibilityService : AccessibilityService(), LifecycleOwner, IAccessib } val strokeDescription = - when { - inputEventType == InputEventType.DOWN && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O -> - StrokeDescription( - path, - 0, - duration, - true - ) - - inputEventType == InputEventType.UP && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O -> - StrokeDescription( - path, - 59999, - duration, - false - ) - - else -> StrokeDescription(path, 0, duration) - } + when { + inputEventType == InputEventType.DOWN && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O -> + StrokeDescription( + path, + 0, + duration, + true + ) + + inputEventType == InputEventType.UP && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O -> + StrokeDescription( + path, + 59999, + duration, + false + ) + + else -> StrokeDescription(path, 0, duration) + } strokeDescription.let { val gestureDescription = GestureDescription.Builder().apply { @@ -407,25 +409,91 @@ class MyAccessibilityService : AccessibilityService(), LifecycleOwner, IAccessib return Error.SdkVersionTooLow(Build.VERSION_CODES.N) } - override fun swipeScreen( - xStart: Int, - yStart: Int, - xEnd: Int, - yEnd: Int, - fingerCount: Int, - duration: Int, - inputEventType: InputEventType - ): Result<*> { - Timber.d( - "ACCESSIBILITY SWIPE SCREEN %d, %d, %d, %d, %s, %d, %s", - xStart, - yStart, - xEnd, - yEnd, - fingerCount, - duration, - inputEventType - ) + override fun fetchAvailableUIElements(): List { + val viewIds = arrayListOf() + + if (rootInActiveWindow != null) { + viewIds.addAll(findViewIdResourceNames(rootInActiveWindow)) + } else { + Timber.d("fetchAvailableUIElements NO ROOT WINDOW") + } + + val sorted = viewIds.distinct().sorted() + + return sorted.ifEmpty { + emptyList() + } + } + + private fun findViewIdResourceNames(node: AccessibilityNodeInfo): List { + val list = arrayListOf() + + for (i in 0 until node.childCount) { + val child = node.getChild(i) + + if (child != null) { + try { + if (child.viewIdResourceName != null) { + list.add(child.viewIdResourceName) + } + } catch (error: kotlin.Error) { + Timber.d("Could not add child to list: %s", error.message) + } + + list.addAll(findViewIdResourceNames(child)) + } + } + + return list + } + + private fun findNodeByFullyQualifiedName(name: String, parent: AccessibilityNodeInfo): AccessibilityNodeInfo? { + val nodes = arrayListOf() + var result: AccessibilityNodeInfo? = null + + if (rootInActiveWindow != null) { + nodes.addAll(getChildNodes(name, parent)) + + if (nodes.isNotEmpty()) { + nodes.forEach{ + if (it.viewIdResourceName !== null && it.viewIdResourceName == name) { + result = it + return@forEach + } + } + } + } else { + Timber.d("findNodeByFullyQualifiedName NO ROOT WINDOW") + } + + return result + } + + private fun getChildNodes(name: String, parent: AccessibilityNodeInfo): List { + val list = arrayListOf() + + for (i in 0 until parent.childCount) { + val child = parent.getChild(i) + + if (child != null) { + try { + if (child.viewIdResourceName != null) { + list.add(child) + if (child.viewIdResourceName == name) return list + } + } catch (error: kotlin.Error) { + Timber.d("Could not add child to list: %s", error.message) + } + + list.addAll(getChildNodes(name, child)) + } + } + + return list + } + + override fun swipeScreen(xStart: Int, yStart: Int, xEnd: Int, yEnd: Int, fingerCount: Int, duration: Int, inputEventType: InputEventType): Result<*> { + Timber.d("ACCESSIBILITY SWIPE SCREEN %d, %d, %d, %d, %s, %d, %s", xStart, yStart, xEnd, yEnd, fingerCount, duration, inputEventType) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { val pStart = Point(xStart, yStart) @@ -448,19 +516,13 @@ class MyAccessibilityService : AccessibilityService(), LifecycleOwner, IAccessib // the length of each segment between fingers val segmentLength = perpendicularLineLength / segmentCount // perpendicular line of the start swipe point - val perpendicularLineStart = getPerpendicularOfLine( - pStart, pEnd, + val perpendicularLineStart = getPerpendicularOfLine(pStart, pEnd, perpendicularLineLength ) // perpendicular line of the end swipe point - val perpendicularLineEnd = getPerpendicularOfLine( - pEnd, pStart, - perpendicularLineLength, true - ) - + val perpendicularLineEnd = getPerpendicularOfLine(pEnd, pStart, + perpendicularLineLength, true) - val startFingerCoordinatesList = mutableListOf() - val endFingerCoordinatesList = mutableListOf() // this is the angle between start and end point to rotate all virtual fingers on the perpendicular lines in the same direction val angle = angleBetweenPoints(Point(xStart, yStart), Point(xEnd, yEnd)) - 90 @@ -469,19 +531,15 @@ class MyAccessibilityService : AccessibilityService(), LifecycleOwner, IAccessib // offset of each finger val fingerOffsetLength = index * segmentLength * 2 // move the coordinates of the current virtual finger on the perpendicular line for the start coordinates - val startFingerCoordinateWithOffset = - movePointByDistanceAndAngle(perpendicularLineStart.start, fingerOffsetLength, angle) + val startFingerCoordinateWithOffset = movePointByDistanceAndAngle(perpendicularLineStart.start, fingerOffsetLength, angle) // move the coordinates of the current virtual finger on the perpendicular line for the end coordinates - val endFingerCoordinateWithOffset = - movePointByDistanceAndAngle(perpendicularLineEnd.start, fingerOffsetLength, angle) + val endFingerCoordinateWithOffset = movePointByDistanceAndAngle(perpendicularLineEnd.start, fingerOffsetLength, angle) // create a path for each finger, move the the coordinates on the perpendicular line and draw it to the end coordinates of the perpendicular line of the end swipe point val p = Path() p.moveTo(startFingerCoordinateWithOffset.x.toFloat(), startFingerCoordinateWithOffset.y.toFloat()) p.lineTo(endFingerCoordinateWithOffset.x.toFloat(), endFingerCoordinateWithOffset.y.toFloat()) - //startFingerCoordinatesList.add(startFingerCoordinateWithOffset) - //endFingerCoordinatesList.add(endFingerCoordinateWithOffset) gestureBuilder.addStroke(StrokeDescription(p, 0, duration.toLong())) } @@ -499,6 +557,54 @@ class MyAccessibilityService : AccessibilityService(), LifecycleOwner, IAccessib return Error.SdkVersionTooLow(Build.VERSION_CODES.N) } + override fun interactWithScreenElement(fullName: String, onlyIfVisible: Boolean, interactiontype: INTERACTIONTYPE, inputEventType: InputEventType): Result<*> { + + Timber.d("interactWithScreenElement fullName: %s, onlyIfVisible: %s, interactionType: %s", fullName, onlyIfVisible.toString(), interactiontype.name) + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + if (rootInActiveWindow != null) { + // TODO: Find a way to get the "SystemView" as "rootInActiveWindow" + // Use a custom function because "findAccessibilityNodeInfosByViewId" does not iterate through all children sometimes + val nodeToInteractWith = findNodeByFullyQualifiedName(fullName, rootInActiveWindow) + + if (nodeToInteractWith != null) { + if (onlyIfVisible && !nodeToInteractWith.isVisibleToUser) { + return Error.AccessibilityNodeNotVisible + } + + val success = nodeToInteractWith.performAction( + when (interactiontype) { + INTERACTIONTYPE.LONG_CLICK -> AccessibilityNodeInfo.ACTION_LONG_CLICK + INTERACTIONTYPE.SELECT -> AccessibilityNodeInfo.ACTION_SELECT + INTERACTIONTYPE.FOCUS -> AccessibilityNodeInfo.ACTION_FOCUS + INTERACTIONTYPE.CLEAR_FOCUS -> AccessibilityNodeInfo.ACTION_CLEAR_FOCUS + INTERACTIONTYPE.COLLAPSE -> AccessibilityNodeInfo.ACTION_COLLAPSE + INTERACTIONTYPE.EXPAND -> AccessibilityNodeInfo.ACTION_EXPAND + INTERACTIONTYPE.DISMISS -> AccessibilityNodeInfo.ACTION_DISMISS + INTERACTIONTYPE.SCROLL_FORWARD -> AccessibilityNodeInfo.ACTION_SCROLL_FORWARD + INTERACTIONTYPE.SCROLL_BACKWARD -> AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD + else -> AccessibilityNodeInfo.ACTION_CLICK + } + ) + + if (success) { + return Success(Unit) + } else { + Error.FailedToDispatchGesture + } + + } else { + Timber.d("interactWithScreenElement: nodeToInteractWith is null") + return Error.FailedToFindAccessibilityNode + } + } else { + Timber.d("interactWithScreenElement: rootInActiveWindow is NULL") + } + } + + return Error.SdkVersionTooLow(Build.VERSION_CODES.N) + } + override fun findFocussedNode(focus: Int): AccessibilityNodeModel? { return findFocus(focus)?.toModel() } diff --git a/app/src/main/java/io/github/sds100/keymapper/system/apps/ChooseAppViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/system/apps/ChooseAppViewModel.kt index ae195ee2c8..3570bb4e4f 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/apps/ChooseAppViewModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/apps/ChooseAppViewModel.kt @@ -13,6 +13,7 @@ import io.github.sds100.keymapper.util.valueOrNull import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.* import kotlinx.coroutines.launch +import timber.log.Timber import java.util.* /** @@ -104,6 +105,8 @@ class ChooseAppViewModel constructor( private suspend fun List.buildListItems(): List = flow { forEach { packageInfo -> + Timber.d("buildListItems::packageInfo %s", packageInfo.packageName) + val name = useCase.getAppName(packageInfo.packageName) .valueOrNull() ?: return@forEach diff --git a/app/src/main/java/io/github/sds100/keymapper/system/apps/PackageUtils.kt b/app/src/main/java/io/github/sds100/keymapper/system/apps/PackageUtils.kt index 6669d01e36..111a71ae6b 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/apps/PackageUtils.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/apps/PackageUtils.kt @@ -6,6 +6,11 @@ import android.content.Context * Created by sds100 on 27/10/2018. */ +enum class PACKAGE_INFO_TYPES { + TYPE_PACKAGE_NAME, + TYPE_VIEW_ID +} + object PackageUtils { fun isAppInstalled(ctx: Context, packageName: String): Boolean { @@ -18,4 +23,19 @@ object PackageUtils { return false } } + + fun getInfoFromFullyQualifiedViewName(name: String, infoType: PACKAGE_INFO_TYPES): String? { + val splitted = name.split('/') + + if (splitted.isNotEmpty() && splitted.size == 2) { + if (infoType.name == PACKAGE_INFO_TYPES.TYPE_VIEW_ID.name) { + return splitted[1] + } else if (infoType.name == PACKAGE_INFO_TYPES.TYPE_PACKAGE_NAME.name) { + return splitted[0].replace(":id", "", true ) + } + } + + return null + } + } \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/system/ui/ChooseUiElementFragment.kt b/app/src/main/java/io/github/sds100/keymapper/system/ui/ChooseUiElementFragment.kt new file mode 100644 index 0000000000..0023579774 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/system/ui/ChooseUiElementFragment.kt @@ -0,0 +1,100 @@ +package io.github.sds100.keymapper.system.ui + +import android.view.LayoutInflater +import android.view.ViewGroup +import androidx.fragment.app.viewModels +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.viewModelScope +import androidx.navigation.fragment.navArgs +import com.airbnb.epoxy.EpoxyRecyclerView +import io.github.sds100.keymapper.databinding.FragmentChooseUiElementBinding +import io.github.sds100.keymapper.simple +import io.github.sds100.keymapper.util.Event +import io.github.sds100.keymapper.util.Inject +import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.launchRepeatOnLifecycle +import io.github.sds100.keymapper.util.ui.RecyclerViewFragment +import io.github.sds100.keymapper.util.ui.RecyclerViewUtils +import io.github.sds100.keymapper.util.ui.SimpleListItem +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.collectLatest +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.launch +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json + +class ChooseUiElementFragment : RecyclerViewFragment() { + companion object { + const val EXTRA_UI_ELEMENT_ID = "extra_ui_element_id" + const val SEARCH_STATE_KEY = "key_ui_element_search_state" + } + + private val args: ChooseUiElementFragmentArgs by navArgs() + + override var searchStateKey: String? = SEARCH_STATE_KEY + + private val viewModel: ChooseUiElementViewModel by viewModels { + Inject.chooseUiElementViewModel(requireContext()) + } + + override val listItems: Flow>> + get() = viewModel.state.map { it.listItems } + + override fun subscribeUi(binding: FragmentChooseUiElementBinding) { + binding.viewModel = viewModel + + viewLifecycleOwner.launchRepeatOnLifecycle(Lifecycle.State.RESUMED) { + viewModel.returnResult.collectLatest { + returnResult(EXTRA_UI_ELEMENT_ID to Json.encodeToString(it)) + } + } + + RecyclerViewUtils.applySimpleListItemDecorations(binding.epoxyRecyclerView) + } + + override fun onSearchQuery(query: String?) { + viewModel.searchQuery.value = query + } + + override fun bind(inflater: LayoutInflater, container: ViewGroup?) = + FragmentChooseUiElementBinding.inflate(inflater, container, false).apply { + lifecycleOwner = viewLifecycleOwner + } + + override fun populateList(recyclerView: EpoxyRecyclerView, listItems: List) { + binding.epoxyRecyclerView.withModels { + listItems.forEach { + simple { + id(it.id) + model(it) + + onClickListener { _ -> + viewModel.onListItemClick(it.id) + } + } + } + } + } + + override fun getRequestKey(): String { + return args.requestKey + } + + override fun getBottomAppBar(binding: FragmentChooseUiElementBinding) = binding.appBar + override fun getRecyclerView(binding: FragmentChooseUiElementBinding) = binding.epoxyRecyclerView + override fun getProgressBar(binding: FragmentChooseUiElementBinding) = binding.progressBar + override fun getEmptyListPlaceHolderTextView(binding: FragmentChooseUiElementBinding) = binding.emptyListPlaceHolder + + override fun onDestroy() { + super.onDestroy() + + viewModel.stopRecording() + } + + override fun onBackPressed() { + super.onBackPressed() + + viewModel.stopRecording() + } +} \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/system/ui/ChooseUiElementViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/system/ui/ChooseUiElementViewModel.kt new file mode 100644 index 0000000000..ce55b3c587 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/system/ui/ChooseUiElementViewModel.kt @@ -0,0 +1,198 @@ +package io.github.sds100.keymapper.system.ui + +import androidx.lifecycle.ViewModel +import androidx.lifecycle.ViewModelProvider +import androidx.lifecycle.viewModelScope +import io.github.sds100.keymapper.R +import io.github.sds100.keymapper.system.accessibility.ServiceAdapter +import io.github.sds100.keymapper.system.apps.DisplayAppsUseCase +import io.github.sds100.keymapper.system.apps.PACKAGE_INFO_TYPES +import io.github.sds100.keymapper.system.apps.PackageUtils +import io.github.sds100.keymapper.util.Event +import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.filterByQuery +import io.github.sds100.keymapper.util.formatSeconds +import io.github.sds100.keymapper.util.mapData +import io.github.sds100.keymapper.util.ui.DefaultSimpleListItem +import io.github.sds100.keymapper.util.ui.IconInfo +import io.github.sds100.keymapper.util.ui.ResourceProvider +import io.github.sds100.keymapper.util.ui.SimpleListItem +import io.github.sds100.keymapper.util.valueOrNull +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.collectLatest +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.launch +import timber.log.Timber +import java.util.Locale + +class ChooseUiElementViewModel constructor( + useCase: DisplayUiElementsUseCase, + resourceProvider: ResourceProvider, + recordUiElements: RecordUiElementsUseCase, + serviceAdapter: ServiceAdapter, + displayAppsUseCase: DisplayAppsUseCase, +) : ViewModel(), + ResourceProvider by resourceProvider, + DisplayAppsUseCase by displayAppsUseCase +{ + + private val _serviceAdapter = serviceAdapter + + val searchQuery = MutableStateFlow(null) + + private val _state = MutableStateFlow( + UiElementsListState(State.Loading) + ) + val state = _state.asStateFlow() + + private val allAppListItems = useCase.uiElements.map { state -> + state.mapData { uiElements -> + uiElements.buildListItems() + } + }.flowOn(Dispatchers.Default) + + private val _returnResult = MutableSharedFlow() + val returnResult = _returnResult.asSharedFlow() + + val recordButtonText: StateFlow = recordUiElements.state.map { recordUiElementsState -> + when (recordUiElementsState) { + is RecordUiElementsState.CountingDown -> getString(R.string.button_label_choose_ui_element_record_button_text_active, formatSeconds(recordUiElementsState.timeLeft)) + is RecordUiElementsState.Stopped -> getString(R.string.button_label_choose_ui_element_record_button_text_start) + } + }.flowOn(Dispatchers.Default).stateIn(viewModelScope, SharingStarted.Eagerly, getString(R.string.button_label_choose_ui_element_record_button_text_start)) + + val recordDescriptionText: StateFlow = recordUiElements.state.map { recordUiElementsState -> + when (recordUiElementsState) { + is RecordUiElementsState.CountingDown -> getString(R.string.extra_label_interact_with_screen_element_record_description_text_active) + is RecordUiElementsState.Stopped -> getString(R.string.extra_label_interact_with_screen_element_record_description_text_start) + } + }.flowOn(Dispatchers.Default).stateIn(viewModelScope, SharingStarted.Eagerly, getString(R.string.extra_label_interact_with_screen_element_record_description_text_start)) + + private val _isRecording: StateFlow = recordUiElements.state.map { recordUiElementsState -> + when (recordUiElementsState) { + is RecordUiElementsState.CountingDown -> true + is RecordUiElementsState.Stopped -> false + } + }.flowOn(Dispatchers.Default).stateIn(viewModelScope, SharingStarted.Eagerly, false) + + fun stopRecording() { + viewModelScope.launch(Dispatchers.Default) { + _serviceAdapter.send(Event.StopRecordingUiElements) + } + } + + init { + + viewModelScope.launch { + useCase.updateUiElementsList() + } + + combine( + allAppListItems, + searchQuery + ) { allAppListItems, query -> + when (allAppListItems) { + is State.Data -> { + allAppListItems.data.filterByQuery(query).collectLatest { filteredListItems -> + _state.value = UiElementsListState(filteredListItems) + } + } + + is State.Loading -> _state.value = UiElementsListState(State.Loading) + } + }.launchIn(viewModelScope) + } + + fun onListItemClick(id: String) { + Timber.d("onListItemClick: %s", id) + + stopRecording() + + val elementViewId = PackageUtils.getInfoFromFullyQualifiedViewName(id, PACKAGE_INFO_TYPES.TYPE_VIEW_ID) + val elementPackageName = PackageUtils.getInfoFromFullyQualifiedViewName(id, PACKAGE_INFO_TYPES.TYPE_PACKAGE_NAME) + + if (elementViewId != null && elementPackageName != null) { + viewModelScope.launch { + _returnResult.emit( + UiElementInfo( + elementName = elementViewId, + packageName = elementPackageName, + fullName = id, + appName = getAppName(elementPackageName).valueOrNull() + ) + ) + } + } + } + + fun onRecordButtonClick() { + if (_isRecording.value) { + viewModelScope.launch(Dispatchers.Default) { + _serviceAdapter.send(Event.StopRecordingUiElements) + } + } else { + viewModelScope.launch(Dispatchers.Default) { + _serviceAdapter.send(Event.StartRecordingUiElements) + } + } + } + + fun onClearListButtonClick() { + viewModelScope.launch(Dispatchers.Default) { + _serviceAdapter.send(Event.StopRecordingUiElements) + _serviceAdapter.send(Event.ClearRecordedUiElements) + } + } + + private suspend fun List.buildListItems(): List = flow { + + forEach { uiElementInfo -> + val icon = getAppIcon(uiElementInfo.packageName).valueOrNull() + + val listItem = UiElementInfoListItem( + id = uiElementInfo.fullName, + title = uiElementInfo.elementName, + subtitle = uiElementInfo.appName ?: uiElementInfo.packageName, + icon = if (icon != null) IconInfo(icon) else null + ) + + emit(listItem) + } + }.flowOn(Dispatchers.Default).toList().sortedBy { it.id.lowercase(Locale.getDefault()) } + + override fun onCleared() { + super.onCleared() + + stopRecording() + } + + class Factory( + private val useCase: DisplayUiElementsUseCase, + private val resourceProvider: ResourceProvider, + private val recordUiElements: RecordUiElementsUseCase, + private val serviceAdapter: ServiceAdapter, + private val displayAppsUseCase: DisplayAppsUseCase, + ) : ViewModelProvider.Factory { + + @Suppress("UNCHECKED_CAST") + override fun create(modelClass: Class) = + ChooseUiElementViewModel(useCase, resourceProvider, recordUiElements, serviceAdapter, displayAppsUseCase) as T + } +} + + +data class UiElementsListState( + val listItems: State> +) diff --git a/app/src/main/java/io/github/sds100/keymapper/system/ui/DisplayUiElementsUseCase.kt b/app/src/main/java/io/github/sds100/keymapper/system/ui/DisplayUiElementsUseCase.kt new file mode 100644 index 0000000000..c013a93bed --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/system/ui/DisplayUiElementsUseCase.kt @@ -0,0 +1,50 @@ +package io.github.sds100.keymapper.system.ui + +import io.github.sds100.keymapper.data.repositories.ViewIdRepository +import io.github.sds100.keymapper.mappings.keymaps.DisplayKeyMapUseCase +import io.github.sds100.keymapper.mappings.keymaps.KeyMapActionUiHelper +import io.github.sds100.keymapper.system.apps.DisplayAppsUseCase +import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.dataOrNull +import io.github.sds100.keymapper.util.valueOrNull +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.collectLatest +import kotlinx.coroutines.withContext + +class DisplayUiElementsUseCaseImpl ( + private val viewIdRepository: ViewIdRepository, + displayApps: DisplayAppsUseCase, +): DisplayUiElementsUseCase { + private val actionUiHelper by lazy { displayApps } + override val uiElements: MutableStateFlow>> = MutableStateFlow(State.Loading) + + override suspend fun updateUiElementsList() { + withContext(Dispatchers.Default) { + uiElements.value = State.Loading + + viewIdRepository.viewIdList.collectLatest { data -> + val tmpList = arrayListOf() + + data.dataOrNull()?.map { viewIdEntity -> + tmpList.add( + UiElementInfo( + elementName = viewIdEntity.viewId, + packageName = viewIdEntity.packageName, + fullName = viewIdEntity.fullName, + appName = actionUiHelper.getAppName(viewIdEntity.packageName).valueOrNull() + ) + ) + } + + uiElements.value = State.Data(tmpList) + } + } + } +} + +interface DisplayUiElementsUseCase { + val uiElements: MutableStateFlow>> + + suspend fun updateUiElementsList() +} \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/system/ui/RecordUiElementsState.kt b/app/src/main/java/io/github/sds100/keymapper/system/ui/RecordUiElementsState.kt new file mode 100644 index 0000000000..9fa1d6909a --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/system/ui/RecordUiElementsState.kt @@ -0,0 +1,12 @@ +package io.github.sds100.keymapper.system.ui + +sealed class RecordUiElementsState { + data class CountingDown( + /** + * The time left in seconds + */ + val timeLeft: Int + ) : RecordUiElementsState() + + object Stopped : RecordUiElementsState() +} \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/system/ui/RecordUiElementsUseCase.kt b/app/src/main/java/io/github/sds100/keymapper/system/ui/RecordUiElementsUseCase.kt new file mode 100644 index 0000000000..a980634f48 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/system/ui/RecordUiElementsUseCase.kt @@ -0,0 +1,43 @@ +package io.github.sds100.keymapper.system.ui + +import io.github.sds100.keymapper.system.accessibility.ServiceAdapter +import io.github.sds100.keymapper.util.Event +import io.github.sds100.keymapper.util.Result +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.* + +class RecordUiElementsController( + private val coroutineScope: CoroutineScope, + private val serviceAdapter: ServiceAdapter +) : RecordUiElementsUseCase { + override val state = MutableStateFlow(RecordUiElementsState.Stopped) + + init { + serviceAdapter.eventReceiver.onEach { event -> + when (event) { + is Event.OnStoppedRecordingUiElements -> state.value = RecordUiElementsState.Stopped + is Event.OnIncrementRecordUiElementsTimer -> state.value = + RecordUiElementsState.CountingDown(event.timeLeft) + else -> Unit + } + }.launchIn(coroutineScope) + } + + override suspend fun startRecording(): Result<*> { + return serviceAdapter.send(Event.StartRecordingUiElements) + } + + override suspend fun stopRecording(): Result<*> { + return serviceAdapter.send(Event.StopRecordingUiElements) + } +} + +interface RecordUiElementsUseCase { + val state: Flow + + /** + * @return Success if started and an Error if failed to start. + */ + suspend fun startRecording(): Result<*> + suspend fun stopRecording(): Result<*> +} \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/system/ui/UiElementInfo.kt b/app/src/main/java/io/github/sds100/keymapper/system/ui/UiElementInfo.kt new file mode 100644 index 0000000000..03f49ce0ba --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/system/ui/UiElementInfo.kt @@ -0,0 +1,11 @@ +package io.github.sds100.keymapper.system.ui + +import kotlinx.serialization.Serializable + +@Serializable +data class UiElementInfo ( + val elementName: String, + val packageName: String, + val fullName: String, + val appName: String? +) \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/system/ui/UiElementInfoListItem.kt b/app/src/main/java/io/github/sds100/keymapper/system/ui/UiElementInfoListItem.kt new file mode 100644 index 0000000000..9306b35f1e --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/system/ui/UiElementInfoListItem.kt @@ -0,0 +1,18 @@ +package io.github.sds100.keymapper.system.ui + +import io.github.sds100.keymapper.util.ui.IconInfo +import io.github.sds100.keymapper.util.ui.SimpleListItem +import io.github.sds100.keymapper.util.ui.TintType + +class UiElementInfoListItem ( + override val id: String, + override val title: String, + override val subtitle: String?, + override val subtitleTint: TintType = TintType.None, + override val icon: IconInfo?, + override val isEnabled: Boolean = true +): SimpleListItem { + override fun getSearchableString(): String { + return id + } +} \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ErrorUtils.kt b/app/src/main/java/io/github/sds100/keymapper/util/ErrorUtils.kt index 6ef38b075b..4934d8107d 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ErrorUtils.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ErrorUtils.kt @@ -75,6 +75,7 @@ fun Error.getFullMessage(resourceProvider: ResourceProvider) = when (this) { Error.CantFindImeSettings -> resourceProvider.getString(R.string.error_cant_find_ime_settings) Error.CantShowImePickerInBackground -> resourceProvider.getString(R.string.error_cant_show_ime_picker_in_background) Error.FailedToFindAccessibilityNode -> resourceProvider.getString(R.string.error_failed_to_find_accessibility_node) + Error.AccessibilityNodeNotVisible -> resourceProvider.getString(R.string.error_accessibility_node_not_visible) is Error.FailedToPerformAccessibilityGlobalAction -> resourceProvider.getString( R.string.error_failed_to_perform_accessibility_global_action, action diff --git a/app/src/main/java/io/github/sds100/keymapper/util/Event.kt b/app/src/main/java/io/github/sds100/keymapper/util/Event.kt index ea89caf11d..85a6096395 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/Event.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/Event.kt @@ -78,4 +78,19 @@ sealed class Event { @Serializable data class OnInputFocusChange(val isFocussed: Boolean) : Event() + + @Serializable + object StartRecordingUiElements: Event() + + @Serializable + object StopRecordingUiElements: Event() + + @Serializable + data class OnIncrementRecordUiElementsTimer(val timeLeft: Int) : Event() + + @Serializable + object OnStoppedRecordingUiElements : Event() + + @Serializable + object ClearRecordedUiElements: Event() } \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt b/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt index 39434975be..3f18599b22 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt @@ -10,6 +10,7 @@ import io.github.sds100.keymapper.actions.TestActionUseCaseImpl import io.github.sds100.keymapper.actions.keyevent.ChooseKeyCodeViewModel import io.github.sds100.keymapper.actions.keyevent.ConfigKeyEventActionViewModel import io.github.sds100.keymapper.actions.keyevent.ConfigKeyEventUseCaseImpl +import io.github.sds100.keymapper.actions.uielementinteraction.InteractWithScreenElementViewModel import io.github.sds100.keymapper.actions.sound.ChooseSoundFileUseCaseImpl import io.github.sds100.keymapper.actions.sound.ChooseSoundFileViewModel import io.github.sds100.keymapper.actions.swipescreen.SwipePickDisplayCoordinateViewModel @@ -43,6 +44,7 @@ import io.github.sds100.keymapper.system.apps.DisplayAppShortcutsUseCaseImpl import io.github.sds100.keymapper.system.bluetooth.ChooseBluetoothDeviceUseCaseImpl import io.github.sds100.keymapper.system.bluetooth.ChooseBluetoothDeviceViewModel import io.github.sds100.keymapper.system.intents.ConfigIntentViewModel +import io.github.sds100.keymapper.system.ui.ChooseUiElementViewModel /** * Created by sds100 on 26/01/2020. @@ -90,6 +92,16 @@ object Inject { ) } + fun chooseUiElementViewModel(context: Context): ChooseUiElementViewModel.Factory { + return ChooseUiElementViewModel.Factory( + UseCases.displayUiElements(context), + ServiceLocator.resourceProvider(context), + (context.applicationContext as KeyMapperApp).recordUiElementsController, + ServiceLocator.accessibilityServiceAdapter(context), + UseCases.displayPackages(context) + ) + } + fun configKeyEventViewModel( context: Context ): ConfigKeyEventActionViewModel.Factory { @@ -132,6 +144,12 @@ object Inject { ServiceLocator.resourceProvider(context) ) } + fun interactWithScreenElementActionTypeViewModel(context: Context): InteractWithScreenElementViewModel.Factory { + return InteractWithScreenElementViewModel.Factory( + ServiceLocator.resourceProvider(context), + UseCases.displayPackages(context) + ) + } fun configKeyMapViewModel( ctx: Context @@ -274,7 +292,8 @@ object Inject { suAdapter = ServiceLocator.suAdapter(service), rerouteKeyEventsUseCase = UseCases.rerouteKeyEvents(service), inputMethodAdapter = ServiceLocator.inputMethodAdapter(service), - settingsRepository = ServiceLocator.settingsRepository(service) + settingsRepository = ServiceLocator.settingsRepository(service), + viewIdRepository = ServiceLocator.viewIdRepository(service) ) } diff --git a/app/src/main/java/io/github/sds100/keymapper/util/MathUtils.kt b/app/src/main/java/io/github/sds100/keymapper/util/MathUtils.kt index 8fad6c604c..fa87ee6d33 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/MathUtils.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/MathUtils.kt @@ -12,11 +12,11 @@ data class Line( ) fun deg2rad(degrees: Double): Double { - return degrees * Math.PI / 180 + return degrees * Math.PI / 180; } fun rad2deg(radians: Double): Double { - return radians * 180 / Math.PI + return radians * 180 / Math.PI; } fun getPerpendicularOfLine( @@ -43,8 +43,8 @@ fun getPerpendicularOfLine( } fun movePointByDistanceAndAngle(p: Point, distance: Int, degrees: Double): Point { - val newX = (p.x + cos(deg2rad(degrees)) * distance).toInt() - val newY = (p.y + sin(deg2rad(degrees)) * distance).toInt() + val newX = (p.x + cos(deg2rad(degrees)) * distance).toInt(); + val newY = (p.y + sin(deg2rad(degrees)) * distance).toInt(); return Point(newX, newY) } diff --git a/app/src/main/java/io/github/sds100/keymapper/util/Result.kt b/app/src/main/java/io/github/sds100/keymapper/util/Result.kt index cd55da3b9a..d0bfa542a0 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/Result.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/Result.kt @@ -84,6 +84,9 @@ sealed class Error : Result() { } object FailedToFindAccessibilityNode : Error() + + object AccessibilityNodeNotVisible: Error() + data class FailedToPerformAccessibilityGlobalAction(val action: Int) : Error() object FailedToDispatchGesture : Error() diff --git a/app/src/main/java/io/github/sds100/keymapper/util/TimeUtils.kt b/app/src/main/java/io/github/sds100/keymapper/util/TimeUtils.kt new file mode 100644 index 0000000000..8b4f179f22 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/util/TimeUtils.kt @@ -0,0 +1,7 @@ +package io.github.sds100.keymapper.util + +fun formatSeconds(seconds: Int, divider: String = ":"): String { + val min = (seconds / 60).toString().padStart(2, '0').padEnd(2, '0') + val sec = (seconds % 60).toString().padStart(2, '0').padEnd(2, '0') + return "$min$divider$sec" +} \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/NavDestination.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/NavDestination.kt index 62ee45ed6e..07cdd81221 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ui/NavDestination.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/NavDestination.kt @@ -4,6 +4,7 @@ import io.github.sds100.keymapper.actions.ActionData import io.github.sds100.keymapper.actions.sound.ChooseSoundFileResult import io.github.sds100.keymapper.actions.swipescreen.SwipePickCoordinateResult import io.github.sds100.keymapper.actions.tapscreen.PickCoordinateResult +import io.github.sds100.keymapper.actions.uielementinteraction.InteractWithScreenElementResult import io.github.sds100.keymapper.constraints.ChooseConstraintType import io.github.sds100.keymapper.constraints.Constraint import io.github.sds100.keymapper.mappings.fingerprintmaps.FingerprintMapId @@ -11,6 +12,7 @@ import io.github.sds100.keymapper.system.apps.ActivityInfo import io.github.sds100.keymapper.system.apps.ChooseAppShortcutResult import io.github.sds100.keymapper.system.bluetooth.BluetoothDeviceInfo import io.github.sds100.keymapper.system.intents.ConfigIntentResult +import io.github.sds100.keymapper.system.ui.UiElementInfo import timber.log.Timber /** @@ -24,6 +26,8 @@ sealed class NavDestination { const val ID_KEY_EVENT = "key_event" const val ID_PICK_COORDINATE = "pick_coordinate" const val ID_PICK_SWIPE_COORDINATE = "pick_swipe_coordinate" + const val ID_INTERACT_WITH_SCREEN_ELEMENT = "interact_with_screen_element" + const val ID_CHOOSE_UI_ELEMENT = "choose_ui_element" const val ID_CONFIG_INTENT = "config_intent" const val ID_CHOOSE_ACTIVITY = "choose_activity" const val ID_CHOOSE_SOUND = "choose_sound" @@ -41,21 +45,23 @@ sealed class NavDestination { Timber.d("NavDestination: %s", this.toString()) return when (this) { is ChooseApp -> ID_CHOOSE_APP - ChooseAppShortcut -> ID_CHOOSE_APP_SHORTCUT - ChooseKeyCode -> ID_KEY_CODE + is ChooseAppShortcut -> ID_CHOOSE_APP_SHORTCUT + is ChooseKeyCode -> ID_KEY_CODE is ConfigKeyEventAction -> ID_KEY_EVENT is PickCoordinate -> ID_PICK_COORDINATE is PickSwipeCoordinate -> ID_PICK_SWIPE_COORDINATE + is InteractWithScreenElement -> ID_INTERACT_WITH_SCREEN_ELEMENT + is ChooseUiElement -> ID_CHOOSE_UI_ELEMENT is ConfigIntent -> ID_CONFIG_INTENT - ChooseActivity -> ID_CHOOSE_ACTIVITY - ChooseSound -> ID_CHOOSE_SOUND - ChooseAction -> ID_CHOOSE_ACTION + is ChooseActivity -> ID_CHOOSE_ACTIVITY + is ChooseSound -> ID_CHOOSE_SOUND + is ChooseAction -> ID_CHOOSE_ACTION is ChooseConstraint -> ID_CHOOSE_CONSTRAINT - ChooseBluetoothDevice -> ID_CHOOSE_BLUETOOTH_DEVICE - FixAppKilling -> ID_FIX_APP_KILLING - ReportBug -> ID_REPORT_BUG - Settings -> ID_SETTINGS - About -> ID_ABOUT + is ChooseBluetoothDevice -> ID_CHOOSE_BLUETOOTH_DEVICE + is FixAppKilling -> ID_FIX_APP_KILLING + is ReportBug -> ID_REPORT_BUG + is Settings -> ID_SETTINGS + is About -> ID_ABOUT is ConfigKeyMap -> ID_CONFIG_KEY_MAP is ConfigFingerprintMap -> ID_CONFIG_FINGERPRINT_MAP } @@ -80,6 +86,11 @@ sealed class NavDestination { data class PickSwipeCoordinate(val result: SwipePickCoordinateResult? = null) : NavDestination() + data class InteractWithScreenElement(val result: InteractWithScreenElementResult? = null) : + NavDestination() + + object ChooseUiElement: NavDestination() + data class ConfigIntent(val result: ConfigIntentResult? = null) : NavDestination() diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt index 2d50fa6e8f..54f998144c 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt @@ -18,32 +18,25 @@ import io.github.sds100.keymapper.actions.swipescreen.SwipePickCoordinateResult import io.github.sds100.keymapper.actions.swipescreen.SwipePickDisplayCoordinateFragment import io.github.sds100.keymapper.actions.tapscreen.PickCoordinateResult import io.github.sds100.keymapper.actions.tapscreen.PickDisplayCoordinateFragment +import io.github.sds100.keymapper.actions.uielementinteraction.InteractWithScreenElementFragment +import io.github.sds100.keymapper.actions.uielementinteraction.InteractWithScreenElementResult import io.github.sds100.keymapper.constraints.ChooseConstraintFragment import io.github.sds100.keymapper.constraints.Constraint -import io.github.sds100.keymapper.system.apps.ActivityInfo -import io.github.sds100.keymapper.system.apps.ChooseActivityFragment -import io.github.sds100.keymapper.system.apps.ChooseAppFragment -import io.github.sds100.keymapper.system.apps.ChooseAppShortcutFragment -import io.github.sds100.keymapper.system.apps.ChooseAppShortcutResult +import io.github.sds100.keymapper.system.apps.* import io.github.sds100.keymapper.system.bluetooth.BluetoothDeviceInfo import io.github.sds100.keymapper.system.bluetooth.ChooseBluetoothDeviceFragment import io.github.sds100.keymapper.system.intents.ConfigIntentFragment import io.github.sds100.keymapper.system.intents.ConfigIntentResult +import io.github.sds100.keymapper.system.ui.ChooseUiElementFragment +import io.github.sds100.keymapper.system.ui.UiElementInfo import io.github.sds100.keymapper.ui.utils.getJsonSerializable import io.github.sds100.keymapper.util.ui.NavDestination.Companion.getId -import kotlinx.coroutines.flow.MutableSharedFlow -import kotlinx.coroutines.flow.SharedFlow -import kotlinx.coroutines.flow.asSharedFlow -import kotlinx.coroutines.flow.dropWhile -import kotlinx.coroutines.flow.first -import kotlinx.coroutines.flow.launchIn -import kotlinx.coroutines.flow.map -import kotlinx.coroutines.flow.merge -import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.* import kotlinx.coroutines.runBlocking import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json +import timber.log.Timber /** * Created by sds100 on 25/07/2021. @@ -145,7 +138,7 @@ fun NavigationViewModel.setupNavigation(fragment: Fragment) { } val direction = when (destination) { - is NavDestination.ChooseApp -> NavAppDirections.chooseApp(destination.allowHiddenApps, requestKey) + is NavDestination.ChooseApp -> NavAppDirections.chooseApp(destination.allowHiddenApps, requestKey) NavDestination.ChooseAppShortcut -> NavAppDirections.chooseAppShortcut(requestKey) NavDestination.ChooseKeyCode -> NavAppDirections.chooseKeyCode(requestKey) is NavDestination.ConfigKeyEventAction -> { @@ -155,7 +148,6 @@ fun NavigationViewModel.setupNavigation(fragment: Fragment) { NavAppDirections.configKeyEvent(requestKey, json) } - is NavDestination.PickCoordinate -> { val json = destination.result?.let { Json.encodeToString(it) @@ -163,7 +155,6 @@ fun NavigationViewModel.setupNavigation(fragment: Fragment) { NavAppDirections.pickDisplayCoordinate(requestKey, json) } - is NavDestination.PickSwipeCoordinate -> { val json = destination.result?.let { Json.encodeToString(it) @@ -171,7 +162,14 @@ fun NavigationViewModel.setupNavigation(fragment: Fragment) { NavAppDirections.swipePickDisplayCoordinate(requestKey, json) } + is NavDestination.InteractWithScreenElement -> { + val json = destination.result?.let { + Json.encodeToString(it) + } + NavAppDirections.pickScreenElement(requestKey, json) + } + is NavDestination.ChooseUiElement -> NavAppDirections.chooseUiElement(requestKey) is NavDestination.ConfigIntent -> { val json = destination.result?.let { Json.encodeToString(it) @@ -179,7 +177,6 @@ fun NavigationViewModel.setupNavigation(fragment: Fragment) { NavAppDirections.configIntent(requestKey, json) } - is NavDestination.ChooseActivity -> NavAppDirections.chooseActivity(requestKey) is NavDestination.ChooseSound -> NavAppDirections.chooseSoundFile(requestKey) NavDestination.ChooseAction -> NavAppDirections.toChooseActionFragment(requestKey) @@ -187,11 +184,9 @@ fun NavigationViewModel.setupNavigation(fragment: Fragment) { supportedConstraints = Json.encodeToString(destination.supportedConstraints), requestKey = requestKey ) - is NavDestination.ChooseBluetoothDevice -> NavAppDirections.chooseBluetoothDevice( requestKey ) - NavDestination.FixAppKilling -> NavAppDirections.goToFixAppKillingActivity() NavDestination.ReportBug -> NavAppDirections.goToReportBugActivity() NavDestination.About -> NavAppDirections.actionGlobalAboutFragment() @@ -255,6 +250,19 @@ fun NavigationViewModel.sendNavResultFromBundle( onNavResult(NavResult(requestKey, result)) } + NavDestination.ID_INTERACT_WITH_SCREEN_ELEMENT -> { + val json = bundle.getString(InteractWithScreenElementFragment.EXTRA_RESULT)!! + val result = Json.decodeFromString(json) + onNavResult(NavResult(requestKey, result)) + } + + NavDestination.ID_CHOOSE_UI_ELEMENT -> { + val result = bundle.getJsonSerializable( + ChooseUiElementFragment.EXTRA_UI_ELEMENT_ID + ) + onNavResult(NavResult(requestKey, result)) + } + NavDestination.ID_CONFIG_INTENT -> { val json = bundle.getString(ConfigIntentFragment.EXTRA_RESULT)!! val result = Json.decodeFromString(json) diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/ResourceExt.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/ResourceExt.kt index 49580e40ef..b24a0097da 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ui/ResourceExt.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/ResourceExt.kt @@ -192,3 +192,12 @@ fun Context.obtainStyledAttr(@AttrRes attrRes: Int): TypedArray = if (isMainThre obtainStyledAttributes(cachedAttrArray) } +fun Context.getDynamicStringValue(name: String): String { + val stringId = resources.getIdentifier(name, "string", packageName) + return str(stringId) +} + +fun Fragment.getDynamicStringValue(name: String): String { + val stringId = resources.getIdentifier(name, "string", context?.packageName) + return str(stringId) +} \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/ResourceProvider.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/ResourceProvider.kt index e95444c1de..e819661a84 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ui/ResourceProvider.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/ResourceProvider.kt @@ -7,6 +7,7 @@ import androidx.annotation.DrawableRes import androidx.annotation.StringRes import io.github.sds100.keymapper.util.color import io.github.sds100.keymapper.util.drawable +import io.github.sds100.keymapper.util.getDynamicStringValue import io.github.sds100.keymapper.util.str import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.Flow @@ -49,6 +50,10 @@ class ResourceProviderImpl( return ctx.color(color) } + override fun getDynamicStringValue(name: String): String { + return ctx.getDynamicStringValue(name) + } + fun onThemeChange() { coroutineScope.launch { onThemeChange.emit(Unit) @@ -65,4 +70,6 @@ interface ResourceProvider { fun getText(@StringRes resId: Int): CharSequence fun getDrawable(@DrawableRes resId: Int): Drawable fun getColor(@ColorRes color: Int): Int + + fun getDynamicStringValue(name: String): String } \ No newline at end of file diff --git a/app/src/main/res/drawable/ic_outline_interact_with_screen_element_app_24.xml b/app/src/main/res/drawable/ic_outline_interact_with_screen_element_app_24.xml new file mode 100644 index 0000000000..81e4738a9f --- /dev/null +++ b/app/src/main/res/drawable/ic_outline_interact_with_screen_element_app_24.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/layout/fragment_choose_ui_element.xml b/app/src/main/res/layout/fragment_choose_ui_element.xml new file mode 100644 index 0000000000..61f2b5c3eb --- /dev/null +++ b/app/src/main/res/layout/fragment_choose_ui_element.xml @@ -0,0 +1,136 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/layout/fragment_interact_with_screen_element.xml b/app/src/main/res/layout/fragment_interact_with_screen_element.xml new file mode 100644 index 0000000000..da4921318e --- /dev/null +++ b/app/src/main/res/layout/fragment_interact_with_screen_element.xml @@ -0,0 +1,224 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/navigation/nav_app.xml b/app/src/main/res/navigation/nav_app.xml index 6f801a11b5..def8b58d98 100644 --- a/app/src/main/res/navigation/nav_app.xml +++ b/app/src/main/res/navigation/nav_app.xml @@ -169,6 +169,29 @@ app:argType="string" /> + + + + + + + + + + + + + + + + + + + Tap coordinates %d, %d (%s) Swipe with %d finger(s) from coordinates %d/%d to %d/%d in %dms Swipe with %d finger(s) from coordinates %d/%d to %d/%d in %dms (%s) + Tap screen element with ID %s + Tap screen element with ID %s (%s) Call %s Play sound: %s + Do \"%s\" on element \"%s\" in app \"%s\" + Do \"%s\" on element \"%s\" in app \"%s\" (%s) @@ -175,6 +179,7 @@ Flags Sound file description WiFi network SSID + Description (optional) @@ -256,6 +261,32 @@ Coordinates to set with screenshot Start End + Element ID + Please select an UI element first + Package Name + Please select an UI element first + App Name + No App Selected + Options + Interaction Type + Perform Action + Start Record + Recording for %s min + Clear List + only if visible + Interaction Type + Click + Long Click + Select + Focus + Clear Focus + Collapse + Expand + Dismiss + Scroll Forward + Scroll Backward + Click the record button and switch to the app from where you want to receive UI elements and navigate through the app to fetch as much elements as possible. + You can now switch to the desired app. The recording will automatically stop but you can always resume the recording. @@ -464,6 +495,7 @@ Choose sound file Edit action Replace action + Select UI Element @@ -1024,6 +1056,7 @@ Can\'t find the %s input method The input method picker can\'t be shown! Failed to find accessibility node! + Accessibility node not visible to user! Failed perform global action %s! Your key maps won\'t work! Some things need fixing! @@ -1233,6 +1266,7 @@ Tap screen Swipe screen + Interact with screen element Input text Open URL Send intent diff --git a/app/src/main/res/xml/config_accessibility_service.xml b/app/src/main/res/xml/config_accessibility_service.xml index 820e62eed6..b652b536c5 100644 --- a/app/src/main/res/xml/config_accessibility_service.xml +++ b/app/src/main/res/xml/config_accessibility_service.xml @@ -5,4 +5,9 @@ android:canRequestFingerprintGestures="true" android:canRetrieveWindowContent="true" android:notificationTimeout="0" + android:interactiveUiTimeout="0" + android:nonInteractiveUiTimeout="0" + android:accessibilityFlags="flagDefault|flagRequestFilterKeyEvents|flagIncludeNotImportantViews|flagReportViewIds|flagEnableAccessibilityVolume|flagInputMethodEditor|flagRequestAccessibilityButton|flagRequestEnhancedWebAccessibility|flagRequestFingerprintGestures|flagRequestMultiFingerGestures|flagRequestShortcutWarningDialogSpokenFeedback|flagRequestTouchExplorationMode|flagRetrieveInteractiveWindows|flagSendMotionEvents|flagServiceHandlesDoubleTap" + android:accessibilityEventTypes="typeAllMask" + android:accessibilityFeedbackType="feedbackAllMask" android:description="@string/accessibility_service_explanation" /> \ No newline at end of file diff --git a/app/src/test/java/io/github/sds100/keymapper/mappings/keymaps/ConfigKeyMapTriggerViewModelTest.kt b/app/src/test/java/io/github/sds100/keymapper/mappings/keymaps/ConfigKeyMapTriggerViewModelTest.kt index 408261c96e..f60ea4594a 100644 --- a/app/src/test/java/io/github/sds100/keymapper/mappings/keymaps/ConfigKeyMapTriggerViewModelTest.kt +++ b/app/src/test/java/io/github/sds100/keymapper/mappings/keymaps/ConfigKeyMapTriggerViewModelTest.kt @@ -2,7 +2,7 @@ package io.github.sds100.keymapper.mappings.keymaps import android.view.KeyEvent import io.github.sds100.keymapper.R -import io.github.sds100.keymapper.mappings.keymaps.trigger.RecordTriggerState +import io.github.sds100.keymapper.mappings.keymaps.trigger.RecordUiElementsState import io.github.sds100.keymapper.mappings.keymaps.trigger.RecordTriggerUseCase import io.github.sds100.keymapper.mappings.keymaps.trigger.RecordedKey import io.github.sds100.keymapper.mappings.keymaps.trigger.TriggerKeyDevice @@ -54,7 +54,7 @@ class ConfigKeyMapTriggerViewModelTest { mockRecordTrigger = mock { on { onRecordKey }.then { onRecordKey } - on { state }.then { flow {} } + on { state }.then { flow {} } } mockConfigKeyMapUseCase = mock {