import Cocoa import Foundation typealias Milliseconds = UInt32 extension AXUIElement { var windows: [AXUIElement] { var values: CFArray? AXUIElementCopyAttributeValues(self, kAXWindowsAttribute as CFString, 0, 100, &values) if let values = values { return values as! [AXUIElement] } return [] } var children: [AXUIElement] { var values: CFArray? AXUIElementCopyAttributeValues(self, kAXChildrenAttribute as CFString, 0, 100, &values) if let values = values { return values as! [AXUIElement] } return [] } var groups: [AXUIElement] { return children.filter { $0[kAXRoleAttribute] == kAXGroupRole } } var opaqueGroups: [AXUIElement] { return children.filter { $0[kAXRoleAttribute] == "AXOpaqueProviderGroup" } } var buttons: [AXUIElement] { return children.filter { $0[kAXRoleAttribute] == kAXButtonRole } } var textFields: [AXUIElement] { return children.filter { $0[kAXRoleAttribute] == kAXTextFieldRole} } var staticTexts: [AXUIElement] { return children.filter { $0[kAXRoleAttribute] == kAXStaticTextRole } } var tables: [AXUIElement] { return children.filter { $0[kAXRoleAttribute] == kAXTableRole } } var menuBar: AXUIElement? { return children.first(where: {$0[kAXRoleAttribute] == kAXMenuBarRole }) } var popupButtons: [AXUIElement] { return children.filter { $0[kAXRoleAttribute] == kAXPopUpButtonRole } } var lists: [AXUIElement] { return children.filter { $0[kAXRoleAttribute] == kAXListRole } } var scrollAreas: [AXUIElement] { return children.filter { $0[kAXRoleAttribute] == kAXScrollAreaRole } } var actionNames: [String] { var actionNames: CFArray? let err = AXUIElementCopyActionNames(self, &actionNames) if err == AXError.success, let names = actionNames { return names as! [String] } return [] } /// Search this AXUIElement recursively for the given predicate func search(_ predicate: (AXUIElement) -> Bool) -> AXUIElement? { for child in children { if predicate(child) { return child } if let result = child.search(predicate) { return result } } return nil } func setChildElementValue(title: String, value: String) { guard let element = search({$0[kAXTitleAttribute] == title }) else { print("Failed to find child with title attribute == '\(title)'") return } AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, value as CFString) } var value: String { return self[kAXValueAttribute] ?? "" } func contains(_ value: String) -> Bool { return value.contains(value) } func press() -> Bool { // If we can get the AXEnabled value, check the button is enabled, otherwise, try // to press anyway but print a warning // Added a timeout here in case this is a dialog which has failed // authentication and we're waiting for it to re-enable the button group let buttonEnabled: Bool? = pollForSome(interval: 1000, timeout: .seconds(5)) { if self["AXEnabled"] as Bool? == true { return true } return nil } guard buttonEnabled ?? false else { print("Button is not enabled. It cannot be clicked") return false } // We cannot rely on the result of AXUIElementPerformAction, see documentation _ = AXUIElementPerformAction(self, kAXPressAction as CFString) return true } /// Useful when trying to determine the accessiibility structure of a given dialog /// prints out all attributes for the given AXUIElement to stdout func debug() { var retrieveAtts: CFArray? let err = AXUIElementCopyAttributeNames(self, &retrieveAtts) if err == AXError.success { if let array = retrieveAtts as? [String] { let details = array.reduce(into: [String: String]()) { // $0 in this case will be a CFType, maybe we can make this fully // generic by using CFGetType and correctly casting it to String, Int // Bool, AXValue etc. if let value = self[$1] as String? { $0[$1] = value } else if let value = self[$1] as Int? { $0[$1] = String(describing: value) } else if let value = self[$1] as AXValue? { $0[$1] = String(describing: value) } } print(details as AnyObject) } } } subscript(index: String) -> T? { var value: CFTypeRef? AXUIElementCopyAttributeValue(self, index as CFString, &value) if let value = value { return (value as? T) } return nil } } extension Array where Element == AXUIElement { func contains(_ value: String) -> Bool { for entry in self { if entry.contains(value) { return true } } return false } func search(_ predicate: (AXUIElement) -> Bool) -> AXUIElement? { for entry in self { if let result = entry.search(predicate) { return result } } return nil } subscript(index: String) -> AXUIElement? { let result = self.first(where: { $0[kAXTitleAttribute] == index}) return result as? AXUIElement } } extension CGPoint { static func +(lhs: CGPoint, rhs: CGPoint) -> CGPoint { var result = lhs result.x += rhs.x result.y += rhs.y return result } } extension AXValue { var cgPoint: CGPoint? { switch AXValueGetType(self) { case .cgPoint: var result = CGPoint.zero AXValueGetValue(self, AXValueType.cgPoint, &result) return result default: return nil } } } // Checks if the process is allowed to access accessibility APIs // It's not always the current process that needs access, but normally // the session leader. i.e. for manual runs on your own VM it's likely // to be Terminal.app, where-as for Jenkins agents it's likely to be // /bin/sshd-keygen-wrapper (I've no idea why!) func checkAccess() -> Bool { let checkOptPrompt = kAXTrustedCheckOptionPrompt.takeUnretainedValue() as NSString let options = [checkOptPrompt: true] return AXIsProcessTrustedWithOptions(options as CFDictionary?) } /// Continue to call `f` until it returns some value. Returns `nil` if the timeout occurs. func pollForSome(interval: Milliseconds, timeout: DispatchTimeInterval, f: () -> T?) -> T? { let end = DispatchTime.now() + timeout // os_log("Polling between %d and %d", log: OSLog.scribe, type: .debug, DispatchTime.now().rawValue, end.rawValue) while DispatchTime.now() < end { if let value = f() { return value } usleep(interval * 1000) } return nil } func getRunningApplication(withBundleIdentifier id: String) -> NSRunningApplication? { // Wait for up to 30 seconds for the application to launch return pollForSome(interval: 1000, timeout: .seconds(30)) { let appList = NSRunningApplication.runningApplications(withBundleIdentifier: id) guard appList.count > 0 else { print("No applications with \(id) found") return nil } guard appList[0].isFinishedLaunching else { print("Found app with \(id), but it's not finished launching", id) return nil } print("Application \(id) found") return appList[0] } } guard checkAccess() else { print("You must grant accessibility privileges to this app before using it") exit(EXIT_FAILURE) } guard let app = getRunningApplication(withBundleIdentifier: "com.sampleapp.QtSimpleDialog") else { print("Could not find running application") exit(EXIT_FAILURE) } let appui = AXUIElementCreateApplication(app.processIdentifier) guard let button: AXUIElement = appui.windows[0].buttons["Start Watching"] else { print("Could not find button in app user interface") exit(EXIT_FAILURE) } print("Showing all available attribute/values for button with title \"Start Watching\"") button.debug()