Enhancing Applications with Accessibility API

&OIBODJOH"QQMJDBUJPOTXJUI "DDFTTJCJMJUZ"1* LJTIJLBXBLBUTVNJ !LJTIJLBXBLBUTVNJ!IBDIZEFSNJP LJTIJLBXBLBUTVNJ

IUUQTDPEFWJTVBMTUVEJPDPNVQEBUFTW@@TUSFBNJOHJOMJOFDIBU

IUUQTDPEFWJTVBMTUVEJPDPNVQEBUFTW@@IPMEUPTQFBLNPEF

*OUFHSBUF"*"TTJTUBOUT*OUP"OZ"QQT r *OZPVSGBWPSJUFUFYUFEJUPS r *OZPVSXFCCSPXTFS r 0SDPNJDSFBEFST

4BNQMF$PEF ➡IUUQTHJUIVCDPNLJTIJLBXBLBUTVNJUSZTXJGU

$PSF5FDIOPMPHJFT r 3FUSJFWFTFMFDUFEUFYUGPSBOZBQQMJDBUJPO r .POJUPSHMPCBMLFZNPVTFFWFOUT

"DDFTTJCJMJUZ"1* r "MMPXTBDDFTTUP6*FMFNFOUTPGUIFTZTUFNPSPUIFSBQQMJDBUJPOT r &OBCMFTNBOJQVMBUJPOPG6*FMFNFOUTPGPUIFSBQQMJDBUJPOT

&OBCMJOH"DDFTTJCJMJUZ"1* 4BOECPYBOE1SJWBDZ4FUUJOHT r %JTBCMF4BOECPY r (SBOU"DDFTTJCJMJUZ"1*1FSNJTTJPOT

%JTBCMF4BOECPY

(SBOU"DDFTTJCJMJUZ1FSNJTTJPO 1SJWBDZ4FUUJOHT

(SBOU"DDFTTJCJMJUZ1FSNJTTJPO "9*41SPDFTT5SVTUFE8JUI0QUJPOT @ [kAXTrustedCheckOptionPrompt: true]

(SBOU"DDFTTJCJMJUZ1FSNJTTJPO "9*41SPDFTT5SVTUFE8JUI0QUJPOT @

#BTJD6TBHF "96*&MFNFOU$PQZ"UUSJCVUF7BMVF @@@

#BTJD6TBHF "96*&MFNFOU$PQZ"UUSJCVUF7BMVF @@@ 5IF6*FMFNFOUDPOUBJOJOHUIFWBMVFZPVXBOUUPSFUSJFWF

#BTJD6TBHF "96*&MFNFOU$PQZ"UUSJCVUF7BMVF @@@ ,FZPGUIFWBMVFZPVXBOUUPSFUSJFWF

#BTJD6TBHF "96*&MFNFOU$PQZ"UUSJCVUF7BMVF @@@ 7BMVFZPVXBOUUPSFUSJFWF

#BTJD6TBHF "96*&MFNFOU$PQZ"UUSJCVUF7BMVF @@@ &SSPSDPEF

3FUSJFWF4FMFDUFE5FYU L"94FMFDUFE5FYU"UUSJCVUF var selectedTextValue: AnyObject? let selectedTextValueError = AXUIElementCopyAttributeValue( focusedElement
as! AXUIElement, kAXSelectedTextAttribute as CFString, &selectedTextValue ) guard let selectedTextValue, selectedTextValueError == .success else { return }

4ZTUFN8JEF"96*&MFNFOU "96*&MFNFOU$SFBUF4ZTUFN8JEF let systemWideElement: AXUIElement = AXUIElementCreateSystemWide()

4ZTUFN8JEF"96*&MFNFOU r "DDFTTTZTUFNXJEFLFZCPBSEGPDVT NPVTFDVSTPSQPTJUJPO BOEPUIFS JOGPSNBUJPO r "DDFTTTZTUFNMFWFM6*FMFNFOUTUIBUEPOPUCFMPOHUPBTQFDJ fi D
BQQMJDBUJPO FH NFOVCBS OPUJ fi DBUJPODFOUFS r 3FUSJFWFJOGPSNBUJPOBCPVUUIFDPOUFYUPGUIFDVSSFOUVTFSJOUFSBDUJPO TVDIBTUIFBDUJWFXJOEPXPSUIF6*FMFNFOUUIBUDVSSFOUMZIBTGPDVT

3FUSJFWF'PDVTFE6*&MFNFOU L"9'PDVTFE6*&MFNFOU"UUSJCVUF let systemWideElement = AXUIElementCreateSystemWide() var focusedElement: AnyObject? let
focusedElementError = AXUIElementCopyAttributeValue( systemWideElement, kAXFocusedUIElementAttribute as CFString, &focusedElement )

3FUSJFWF4FMFDUFE5FYU L"94FMFDUFE5FYU"UUSJCVUF var selectedTextValue: AnyObject? let selectedTextValueError = AXUIElementCopyAttributeValue( focusedElement
as! AXUIElement, kAXSelectedTextAttribute as CFString, &selectedTextValue ) guard let selectedTextValue, selectedTextValueError == .success else { return }

.POJUPS(MPCBM,FZ&WFOU /4&WFOUBEE(MPCBM.POJUPS'PS&WFOUT NBUDIJOHIBOEMFS open class NSEvent : NSObject, NSCopying, NSCoding
{ open class func addGlobalMonitorForEvents( matching mask: NSEvent.EventTypeMask, handler block: @escaping (NSEvent) -> Void ) -> Any? }

.POJUPS(MPCBM,FZ&WFOU NSEvent.addGlobalMonitorForEvents( matching: [.keyDown] ) { (event) in switch event.type
{ case .keyDown: guard event.modifierFlags.contains([.command, .control]) && event.keyCode == kVK_ANSI_A else { return } ... default: break } }

.POJUPS(MPCBM,FZ&WFOU $BSCPO"1* *OTUBMM&WFOU)BOEMFS 3FHJTUFS&WFOU)PU,FZ ➡IUUQTHJUIVCDPNLJTIJLBXBLBUTVNJUSZTXJGU r *OTUBMM&WFOU)BOEMFS @@@@@@
r 3FHJTUFS&WFOU)PU,FZ @@@@@@ r $BSCPO"1* r /PBDDFTTJCJMJUZQFSNJTTJPOSFRVJSFE

.POJUPS(MPCBM,FZ&WFOU &WFOU5BQ ➡IUUQTHJUIVCDPNLJTIJLBXBLBUTVNJUSZTXJGU let mask: CGEventMask = (1 << CGEventType.keyDown.rawValue)
let tap = CGEvent.tapCreate( tap: .cgSessionEventTap, place: .headInsertEventTap, options: .defaultTap, eventsOfInterest: mask, callback: { (proxy, type, event, refcon) in ... }, userInfo: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()) ) let runLoopSource = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0) CFRunLoopAddSource(CFRunLoopGetCurrent(), runLoopSource, .commonModes) CGEvent.tapEnable(tap: tap!, enable: true)

.POJUPS(MPCBM,FZ&WFOU ➡IUUQTHJUIVCDPNLJTIJLBXBLBUTVNJUSZTXJGU r $BOTUPQFWFOUQSPQBHBUJPO r $BONPEJGZFWFOU &WFOU5BQ

3FUSJFWF4FMFDUFE5FYU 'SPN8FC7JFX var selectedTextMarkerRangeValue: AnyObject? let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue( focusedElement
as! AXUIElement, "AXSelectedTextMarkerRange" as CFString, &selectedTextMarkerRangeValue ) ... var stringForTextMarkerRangeValue: AnyObject? let stringForTextMarkerRangeValueError = AXUIElementCopyParameterizedAttributeValue( focusedElement as! AXUIElement, "AXStringForTextMarkerRange" as CFString, selectedTextMarkerRangeValue, &stringForTextMarkerRangeValue ) ...

"96*&MFNFOU$PQZ1BSBNFUFSJ[FE"UUSJCVUF7BMVF var selectedTextMarkerRangeValue: AnyObject? let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue( focusedElement as!
AXUIElement, "AXSelectedTextMarkerRange" as CFString, &selectedTextMarkerRangeValue ) ... var stringForTextMarkerRangeValue: AnyObject? let stringForTextMarkerRangeValueError = AXUIElementCopyParameterizedAttributeValue( focusedElement as! AXUIElement, "AXStringForTextMarkerRange" as CFString, selectedTextMarkerRangeValue, &stringForTextMarkerRangeValue ) ...

var selectedTextMarkerRangeValue: AnyObject? let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue( focusedElement as! AXUIElement,
"AXSelectedTextMarkerRange" as CFString, &selectedTextMarkerRangeValue ) ... var stringForTextMarkerRangeValue: AnyObject? let stringForTextMarkerRangeValueError = AXUIElementCopyParameterizedAttributeValue( focusedElement as! AXUIElement, "AXStringForTextMarkerRange" as CFString, selectedTextMarkerRangeValue, &stringForTextMarkerRangeValue ) ... "96*&MFNFOU$PQZ1BSBNFUFSJ[FE"UUSJCVUF7BMVF

)PXUP,OPX6OEPDVNFOUFE,FZT var selectedTextMarkerRangeValue: AnyObject? let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue( focusedElement as!
AXUIElement, "AXSelectedTextMarkerRange" as CFString, &selectedTextMarkerRangeValue ) ... var stringForTextMarkerRangeValue: AnyObject? let stringForTextMarkerRangeValueError = AXUIElementCopyParameterizedAttributeValue( focusedElement as! AXUIElement, "AXStringForTextMarkerRange" as CFString, selectedTextMarkerRangeValue, &stringForTextMarkerRangeValue ) ...

)PXUP,OPX6OEPDVNFOUFE,FZT "96*&MFNFOU$PQZ"UUSJCVUF/BNFT @@

/PUJGJDBUJPOT var observer: AXObserver? let observerCreateError = AXObserverCreate( app.processIdentifier, {
(observer, element, notification, userData) in ... }, &observer ) AXObserverAddNotification( observer, appElement, kAXSelectedTextChangedNotification as CFString, nil ) CFRunLoopAddSource( CFRunLoopGetCurrent(), AXObserverGetRunLoopSource(observer), .commonModes )

(observer, element, notification, userData) in ... }, &observer ) AXObserverAddNotification( observer, appElement, kAXSelectedTextChangedNotification as CFString, nil ) CFRunLoopAddSource( CFRunLoopGetCurrent(), AXObserverGetRunLoopSource(observer), .commonModes ) let app = NSRunningApplication .runningApplications(withBundleIdentifier: "com.apple.TextEdit") .first

(observer, element, notification, userData) in ... }, &observer ) AXObserverAddNotification( observer, appElement, kAXSelectedTextChangedNotification as CFString, nil ) CFRunLoopAddSource( CFRunLoopGetCurrent(), AXObserverGetRunLoopSource(observer), .commonModes )

(observer, element, notification, userData) in ... }, &observer ) AXObserverAddNotification( observer, appElement, kAXSelectedTextChangedNotification as CFString, nil ) CFRunLoopAddSource( CFRunLoopGetCurrent(), AXObserverGetRunLoopSource(observer), .commonModes ) kAXSelectedTextChangedNotification kAXValueChangedNotification ...

.PEJGZ5FYU var isAttributeSettableValue = DarwinBoolean(false) let isAttributeSettableValueError = AXUIElementIsAttributeSettable( element,
kAXValueAttribute as CFString, &isAttributeSettableValue ) guard isAttributeSettableValueError == .success else { return } if isAttributeSettableValue.boolValue { AXUIElementSetAttributeValue( element, kAXValueAttribute as CFString, "\(precedingTextValue)\n\(codeBlock)" as CFString ) return }

4$4DSFFOTIPU.BOBHFSDBQUVSF*NBHF func captureScreen(rect: CGRect) async throws -> CGImage? {
let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false) guard let mainDisplayID = NSScreen.main?.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID, let display = content.displays.first(where: { $0.displayID == mainDisplayID }) else { return nil } let streamConfig = SCStreamConfiguration() streamConfig.captureResolution = .automatic streamConfig.sourceRect = rect streamConfig.preservesAspectRatio = true streamConfig.scalesToFit = true streamConfig.capturesAudio = false streamConfig.excludesCurrentProcessAudio = true let filter = SCContentFilter(display: display, excludingWindows: []) let capturedImage = try await SCScreenshotManager.captureImage( contentFilter: filter, configuration: streamConfig ) return capturedImage }

3FUSJFWFBO*NBHF $(8JOEPX-JTU$SFBUF*NBHF @@@@ func captureScreen(rect: CGRect) -> CGImage? { let
screenShot = CGWindowListCreateImage( rect, [.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID, [.boundsIgnoreFraming, .bestResolution] ) return screenShot }

(SBOU4DSFFO3FDPSEJOH1FSNJTTJPO 1SJWBDZ4FUUJOHT

(SBOU4DSFFO3FDPSEJOH1FSNJTTJPO $(1SF fl JHIU4DSFFO$BQUVSF"DDFTT r $(1SF fl JHIU4DSFFO$BQUVSF"DDFTT r
*OEJDBUFTXIFUIFSTDSFFOSFDPSEJOHJTBMMPXFE r $(3FRVFTU4DSFFO$BQUVSF"DDFTT r 1SFTFOUUIFTZTUFNQSPNQU

&YUSBDUUIF5FYUGSPNBO*NBHF ϒϥοΫδϟοΫʹΑΖ͘͠ ࠤ౻लๆ

7JTJPO,JU ϒϥοΫδϟοΫʹΑΖ͘͠ ࠤ౻लๆ

&YUSBDUUIF5FYUGSPNBO*NBHF *NBHF"OBMZ[FS let analyzer = ImageAnalyzer() let analysis = try
await analyzer.analyze( image, orientation: .up, configuration: configuration ) let transcript = analysis.transcript

8SBQ6Q r 8JOEPX$IBU"TTJTUBOU r #BTJD6TBHFPG"DDFTTJCJMJUZ"1* r *OMJOF$IBU"TTJTUBOU r /PUJ fi
DBUJPOT r .PEJGZ$POUFOU r $PNJD5SBOTMBUPS r 4DSFFO$BQUVSF"1*BOE0$3 r "96*&MFNFOU*OTQFDUPS r *OTQFDU"UUSJCVUF/BNFPG"96*&MFNFOU

4BNQMF$PEF ➡IUUQTHJUIVCDPNLJTIJLBXBLBUTVNJUSZTXJGU

3FGFSFODFT r 4BNQMF$PEF IUUQTHJUIVCDPNLJTIJLBXBLBUTVNJUSZTXJGU r "DDFTTJOHUFYUWBMVFGSPNBOZ4ZTUFNXJEF"QQMJDBUJPOWJB"DDFTTJCJMJUZ "1* IUUQTNBDEFWFMPQFSTXPSEQSFTTDPNBDDFTTJOHUFYUWBMVFGSPNBOZTZTUFNXJEFBQQMJDBUJPOWJB BDDFTTJCJMJUZBQJ r
)PXUPHFUTFMFDUFEUFYUBOEJUTDPPSEJOBUFTGSPNBOZTZTUFNXJEFBQQMJDBUJPOVTJOH "DDFTTJCJMJUZ "1* IUUQTNBDEFWFMPQFSTXPSEQSFTTDPNIPXUPHFUTFMFDUFEUFYUBOEJUTDPPSEJOBUFTGSPNBOZTZTUFNXJEF BQQMJDBUJPOVTJOHBDDFTTJCJMJUZBQJ r 8IZBSFOUUIFNPTUVTFGVM.BDBQQTPOUIF"QQ4UPSF IUUQTBMJOQBOBJUJVDPNCMPHBQQTPVUTJEFBQQTUPSF r "MMBCPVUNBD04FWFOUPCTFSWBUJPO IUUQTEPDTHPPHMFDPNQSFTFOUBUJPOEO&BJ16EVIWKLTS%735D+B&6-C488IU7E()'@946IUNMQSFTFOU

Enhancing Applications with Accessibility API

Enhancing Applications with Accessibility API

More Decks by Kishikawa Katsumi

Other Decks in Programming

Featured

Transcript