Slide 1
Slide 1 text
&OIBODJOH"QQMJDBUJPOTXJUI
"DDFTTJCJMJUZ"1*
LJTIJLBXBLBUTVNJ
!LJTIJLBXBLBUTVNJ!IBDIZEFSNJP
LJTIJLBXBLBUTVNJ
Slide 2
Slide 2 text
IUUQTDPEFWJTVBMTUVEJPDPNVQEBUFTW@@TUSFBNJOHJOMJOFDIBU
Slide 3
Slide 3 text
IUUQTDPEFWJTVBMTUVEJPDPNVQEBUFTW@@IPMEUPTQFBLNPEF
Slide 4
Slide 4 text
*OUFHSBUF"*"TTJTUBOUT*OUP"OZ"QQT
r *OZPVSGBWPSJUFUFYUFEJUPS
r *OZPVSXFCCSPXTFS
r 0SDPNJDSFBEFST
Slide 5
Slide 5 text
4BNQMF$PEF
➡IUUQTHJUIVCDPNLJTIJLBXBLBUTVNJUSZTXJGU
Slide 6
Slide 6 text
No content
Slide 7
Slide 7 text
$PSF5FDIOPMPHJFT
r 3FUSJFWFTFMFDUFEUFYUGPSBOZBQQMJDBUJPO
r .POJUPSHMPCBMLFZNPVTFFWFOUT
Slide 8
Slide 8 text
"DDFTTJCJMJUZ"1*
r "MMPXTBDDFTTUP6*FMFNFOUTPGUIFTZTUFNPSPUIFSBQQMJDBUJPOT
r &OBCMFTNBOJQVMBUJPOPG6*FMFNFOUTPGPUIFSBQQMJDBUJPOT
Slide 9
Slide 9 text
&OBCMJOH"DDFTTJCJMJUZ"1*
4BOECPYBOE1SJWBDZ4FUUJOHT
r %JTBCMF4BOECPY
r (SBOU"DDFTTJCJMJUZ"1*1FSNJTTJPOT
Slide 10
Slide 10 text
%JTBCMF4BOECPY
Slide 11
Slide 11 text
(SBOU"DDFTTJCJMJUZ1FSNJTTJPO
1SJWBDZ4FUUJOHT
Slide 12
Slide 12 text
(SBOU"DDFTTJCJMJUZ1FSNJTTJPO
"9*41SPDFTT5SVTUFE8JUI0QUJPOT @
[kAXTrustedCheckOptionPrompt: true]
Slide 13
Slide 13 text
(SBOU"DDFTTJCJMJUZ1FSNJTTJPO
"9*41SPDFTT5SVTUFE8JUI0QUJPOT @
Slide 14
Slide 14 text
(SBOU"DDFTTJCJMJUZ1FSNJTTJPO
"9*41SPDFTT5SVTUFE8JUI0QUJPOT @
Slide 15
Slide 15 text
(SBOU"DDFTTJCJMJUZ1FSNJTTJPO
"9*41SPDFTT5SVTUFE8JUI0QUJPOT @
Slide 16
Slide 16 text
#BTJD6TBHF
"96*&MFNFOU$PQZ"UUSJCVUF7BMVF @@@
Slide 17
Slide 17 text
#BTJD6TBHF
"96*&MFNFOU$PQZ"UUSJCVUF7BMVF @@@
5IF6*FMFNFOUDPOUBJOJOHUIFWBMVFZPVXBOUUPSFUSJFWF
Slide 18
Slide 18 text
#BTJD6TBHF
"96*&MFNFOU$PQZ"UUSJCVUF7BMVF @@@
,FZPGUIFWBMVFZPVXBOUUPSFUSJFWF
Slide 19
Slide 19 text
#BTJD6TBHF
"96*&MFNFOU$PQZ"UUSJCVUF7BMVF @@@
7BMVFZPVXBOUUPSFUSJFWF
Slide 20
Slide 20 text
#BTJD6TBHF
"96*&MFNFOU$PQZ"UUSJCVUF7BMVF @@@
&SSPSDPEF
Slide 21
Slide 21 text
3FUSJFWF4FMFDUFE5FYU
L"94FMFDUFE5FYU"UUSJCVUF
var selectedTextValue: AnyObject?
let selectedTextValueError = AXUIElementCopyAttributeValue(
focusedElement as! AXUIElement,
kAXSelectedTextAttribute as CFString,
&selectedTextValue
)
guard let selectedTextValue, selectedTextValueError == .success else {
return
}
Slide 22
Slide 22 text
3FUSJFWF4FMFDUFE5FYU
L"94FMFDUFE5FYU"UUSJCVUF
var selectedTextValue: AnyObject?
let selectedTextValueError = AXUIElementCopyAttributeValue(
focusedElement as! AXUIElement,
kAXSelectedTextAttribute as CFString,
&selectedTextValue
)
guard let selectedTextValue, selectedTextValueError == .success else {
return
}
Slide 23
Slide 23 text
3FUSJFWF4FMFDUFE5FYU
L"94FMFDUFE5FYU"UUSJCVUF
var selectedTextValue: AnyObject?
let selectedTextValueError = AXUIElementCopyAttributeValue(
focusedElement as! AXUIElement,
kAXSelectedTextAttribute as CFString,
&selectedTextValue
)
guard let selectedTextValue, selectedTextValueError == .success else {
return
}
Slide 24
Slide 24 text
4ZTUFN8JEF"96*&MFNFOU
"96*&MFNFOU$SFBUF4ZTUFN8JEF
let systemWideElement: AXUIElement = AXUIElementCreateSystemWide()
Slide 25
Slide 25 text
4ZTUFN8JEF"96*&MFNFOU
r "DDFTTTZTUFNXJEFLFZCPBSEGPDVT
NPVTFDVSTPSQPTJUJPO
BOEPUIFS
JOGPSNBUJPO
r "DDFTTTZTUFNMFWFM6*FMFNFOUTUIBUEPOPUCFMPOHUPBTQFDJ
fi
D
BQQMJDBUJPO FH
NFOVCBS
OPUJ
fi
DBUJPODFOUFS
r 3FUSJFWFJOGPSNBUJPOBCPVUUIFDPOUFYUPGUIFDVSSFOUVTFSJOUFSBDUJPO
TVDIBTUIFBDUJWFXJOEPXPSUIF6*FMFNFOUUIBUDVSSFOUMZIBTGPDVT
Slide 26
Slide 26 text
3FUSJFWF'PDVTFE6*&MFNFOU
L"9'PDVTFE6*&MFNFOU"UUSJCVUF
let systemWideElement = AXUIElementCreateSystemWide()
var focusedElement: AnyObject?
let focusedElementError = AXUIElementCopyAttributeValue(
systemWideElement,
kAXFocusedUIElementAttribute as CFString,
&focusedElement
)
Slide 27
Slide 27 text
3FUSJFWF4FMFDUFE5FYU
L"94FMFDUFE5FYU"UUSJCVUF
var selectedTextValue: AnyObject?
let selectedTextValueError = AXUIElementCopyAttributeValue(
focusedElement as! AXUIElement,
kAXSelectedTextAttribute as CFString,
&selectedTextValue
)
guard let selectedTextValue,
selectedTextValueError == .success
else {
return
}
Slide 28
Slide 28 text
No content
Slide 29
Slide 29 text
.POJUPS(MPCBM,FZ&WFOU
/4&WFOUBEE(MPCBM.POJUPS'PS&WFOUT NBUDIJOHIBOEMFS
open class NSEvent : NSObject, NSCopying, NSCoding {
open class func addGlobalMonitorForEvents(
matching mask: NSEvent.EventTypeMask,
handler block: @escaping (NSEvent) -> Void
) -> Any?
}
Slide 30
Slide 30 text
.POJUPS(MPCBM,FZ&WFOU
NSEvent.addGlobalMonitorForEvents(
matching: [.keyDown]
) { (event) in
switch event.type {
case .keyDown:
guard
event.modifierFlags.contains([.command, .control])
&& event.keyCode == kVK_ANSI_A
else {
return
}
...
default:
break
}
}
Slide 31
Slide 31 text
.POJUPS(MPCBM,FZ&WFOU
NSEvent.addGlobalMonitorForEvents(
matching: [.keyDown]
) { (event) in
switch event.type {
case .keyDown:
guard
event.modifierFlags.contains([.command, .control])
&& event.keyCode == kVK_ANSI_A
else {
return
}
...
default:
break
}
}
Slide 32
Slide 32 text
.POJUPS(MPCBM,FZ&WFOU $BSCPO"1*
*OTUBMM&WFOU)BOEMFS
3FHJTUFS&WFOU)PU,FZ
➡IUUQTHJUIVCDPNLJTIJLBXBLBUTVNJUSZTXJGU
r *OTUBMM&WFOU)BOEMFS @@@@@@
r 3FHJTUFS&WFOU)PU,FZ @@@@@@
r $BSCPO"1*
r /PBDDFTTJCJMJUZQFSNJTTJPOSFRVJSFE
Slide 33
Slide 33 text
.POJUPS(MPCBM,FZ&WFOU
&WFOU5BQ
➡IUUQTHJUIVCDPNLJTIJLBXBLBUTVNJUSZTXJGU
let mask: CGEventMask = (1 << CGEventType.keyDown.rawValue)
let tap = CGEvent.tapCreate(
tap: .cgSessionEventTap,
place: .headInsertEventTap,
options: .defaultTap,
eventsOfInterest: mask,
callback: { (proxy, type, event, refcon) in
...
},
userInfo: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque())
)
let runLoopSource = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0)
CFRunLoopAddSource(CFRunLoopGetCurrent(), runLoopSource, .commonModes)
CGEvent.tapEnable(tap: tap!, enable: true)
Slide 34
Slide 34 text
.POJUPS(MPCBM,FZ&WFOU
➡IUUQTHJUIVCDPNLJTIJLBXBLBUTVNJUSZTXJGU
r $BOTUPQFWFOUQSPQBHBUJPO
r $BONPEJGZFWFOU
&WFOU5BQ
Slide 35
Slide 35 text
No content
Slide 36
Slide 36 text
3FUSJFWF4FMFDUFE5FYU 'SPN8FC7JFX
var selectedTextMarkerRangeValue: AnyObject?
let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue(
focusedElement as! AXUIElement,
"AXSelectedTextMarkerRange" as CFString,
&selectedTextMarkerRangeValue
)
...
var stringForTextMarkerRangeValue: AnyObject?
let stringForTextMarkerRangeValueError =
AXUIElementCopyParameterizedAttributeValue(
focusedElement as! AXUIElement,
"AXStringForTextMarkerRange" as CFString,
selectedTextMarkerRangeValue,
&stringForTextMarkerRangeValue
)
...
Slide 37
Slide 37 text
3FUSJFWF4FMFDUFE5FYU 'SPN8FC7JFX
var selectedTextMarkerRangeValue: AnyObject?
let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue(
focusedElement as! AXUIElement,
"AXSelectedTextMarkerRange" as CFString,
&selectedTextMarkerRangeValue
)
...
var stringForTextMarkerRangeValue: AnyObject?
let stringForTextMarkerRangeValueError =
AXUIElementCopyParameterizedAttributeValue(
focusedElement as! AXUIElement,
"AXStringForTextMarkerRange" as CFString,
selectedTextMarkerRangeValue,
&stringForTextMarkerRangeValue
)
...
Slide 38
Slide 38 text
No content
Slide 39
Slide 39 text
"96*&MFNFOU$PQZ1BSBNFUFSJ[FE"UUSJCVUF7BMVF
var selectedTextMarkerRangeValue: AnyObject?
let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue(
focusedElement as! AXUIElement,
"AXSelectedTextMarkerRange" as CFString,
&selectedTextMarkerRangeValue
)
...
var stringForTextMarkerRangeValue: AnyObject?
let stringForTextMarkerRangeValueError =
AXUIElementCopyParameterizedAttributeValue(
focusedElement as! AXUIElement,
"AXStringForTextMarkerRange" as CFString,
selectedTextMarkerRangeValue,
&stringForTextMarkerRangeValue
)
...
Slide 40
Slide 40 text
var selectedTextMarkerRangeValue: AnyObject?
let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue(
focusedElement as! AXUIElement,
"AXSelectedTextMarkerRange" as CFString,
&selectedTextMarkerRangeValue
)
...
var stringForTextMarkerRangeValue: AnyObject?
let stringForTextMarkerRangeValueError =
AXUIElementCopyParameterizedAttributeValue(
focusedElement as! AXUIElement,
"AXStringForTextMarkerRange" as CFString,
selectedTextMarkerRangeValue,
&stringForTextMarkerRangeValue
)
...
"96*&MFNFOU$PQZ1BSBNFUFSJ[FE"UUSJCVUF7BMVF
Slide 41
Slide 41 text
var selectedTextMarkerRangeValue: AnyObject?
let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue(
focusedElement as! AXUIElement,
"AXSelectedTextMarkerRange" as CFString,
&selectedTextMarkerRangeValue
)
...
var stringForTextMarkerRangeValue: AnyObject?
let stringForTextMarkerRangeValueError =
AXUIElementCopyParameterizedAttributeValue(
focusedElement as! AXUIElement,
"AXStringForTextMarkerRange" as CFString,
selectedTextMarkerRangeValue,
&stringForTextMarkerRangeValue
)
...
"96*&MFNFOU$PQZ1BSBNFUFSJ[FE"UUSJCVUF7BMVF
Slide 42
Slide 42 text
)PXUP,OPX6OEPDVNFOUFE,FZT
var selectedTextMarkerRangeValue: AnyObject?
let selectedTextMarkerRangeValueError = AXUIElementCopyAttributeValue(
focusedElement as! AXUIElement,
"AXSelectedTextMarkerRange" as CFString,
&selectedTextMarkerRangeValue
)
...
var stringForTextMarkerRangeValue: AnyObject?
let stringForTextMarkerRangeValueError =
AXUIElementCopyParameterizedAttributeValue(
focusedElement as! AXUIElement,
"AXStringForTextMarkerRange" as CFString,
selectedTextMarkerRangeValue,
&stringForTextMarkerRangeValue
)
...
Slide 43
Slide 43 text
)PXUP,OPX6OEPDVNFOUFE,FZT
"96*&MFNFOU$PQZ"UUSJCVUF/BNFT @@
Slide 44
Slide 44 text
No content
Slide 45
Slide 45 text
No content
Slide 46
Slide 46 text
/PUJGJDBUJPOT
var observer: AXObserver?
let observerCreateError = AXObserverCreate(
app.processIdentifier,
{ (observer, element, notification, userData) in
...
},
&observer
)
AXObserverAddNotification(
observer,
appElement,
kAXSelectedTextChangedNotification as CFString,
nil
)
CFRunLoopAddSource(
CFRunLoopGetCurrent(),
AXObserverGetRunLoopSource(observer),
.commonModes
)
Slide 47
Slide 47 text
/PUJGJDBUJPOT
var observer: AXObserver?
let observerCreateError = AXObserverCreate(
app.processIdentifier,
{ (observer, element, notification, userData) in
...
},
&observer
)
AXObserverAddNotification(
observer,
appElement,
kAXSelectedTextChangedNotification as CFString,
nil
)
CFRunLoopAddSource(
CFRunLoopGetCurrent(),
AXObserverGetRunLoopSource(observer),
.commonModes
)
Slide 48
Slide 48 text
/PUJGJDBUJPOT
var observer: AXObserver?
let observerCreateError = AXObserverCreate(
app.processIdentifier,
{ (observer, element, notification, userData) in
...
},
&observer
)
AXObserverAddNotification(
observer,
appElement,
kAXSelectedTextChangedNotification as CFString,
nil
)
CFRunLoopAddSource(
CFRunLoopGetCurrent(),
AXObserverGetRunLoopSource(observer),
.commonModes
)
let app =
NSRunningApplication
.runningApplications(withBundleIdentifier: "com.apple.TextEdit")
.first
Slide 49
Slide 49 text
/PUJGJDBUJPOT
var observer: AXObserver?
let observerCreateError = AXObserverCreate(
app.processIdentifier,
{ (observer, element, notification, userData) in
...
},
&observer
)
AXObserverAddNotification(
observer,
appElement,
kAXSelectedTextChangedNotification as CFString,
nil
)
CFRunLoopAddSource(
CFRunLoopGetCurrent(),
AXObserverGetRunLoopSource(observer),
.commonModes
)
Slide 50
Slide 50 text
/PUJGJDBUJPOT
var observer: AXObserver?
let observerCreateError = AXObserverCreate(
app.processIdentifier,
{ (observer, element, notification, userData) in
...
},
&observer
)
AXObserverAddNotification(
observer,
appElement,
kAXSelectedTextChangedNotification as CFString,
nil
)
CFRunLoopAddSource(
CFRunLoopGetCurrent(),
AXObserverGetRunLoopSource(observer),
.commonModes
)
kAXSelectedTextChangedNotification
kAXValueChangedNotification
...
Slide 51
Slide 51 text
.PEJGZ5FYU
var isAttributeSettableValue = DarwinBoolean(false)
let isAttributeSettableValueError = AXUIElementIsAttributeSettable(
element,
kAXValueAttribute as CFString,
&isAttributeSettableValue
)
guard isAttributeSettableValueError == .success else {
return
}
if isAttributeSettableValue.boolValue {
AXUIElementSetAttributeValue(
element,
kAXValueAttribute as CFString,
"\(precedingTextValue)\n\(codeBlock)" as CFString
)
return
}
Slide 52
Slide 52 text
.PEJGZ5FYU
var isAttributeSettableValue = DarwinBoolean(false)
let isAttributeSettableValueError = AXUIElementIsAttributeSettable(
element,
kAXValueAttribute as CFString,
&isAttributeSettableValue
)
guard isAttributeSettableValueError == .success else {
return
}
if isAttributeSettableValue.boolValue {
AXUIElementSetAttributeValue(
element,
kAXValueAttribute as CFString,
"\(precedingTextValue)\n\(codeBlock)" as CFString
)
return
}
Slide 53
Slide 53 text
.PEJGZ5FYU
var isAttributeSettableValue = DarwinBoolean(false)
let isAttributeSettableValueError = AXUIElementIsAttributeSettable(
element,
kAXValueAttribute as CFString,
&isAttributeSettableValue
)
guard isAttributeSettableValueError == .success else {
return
}
if isAttributeSettableValue.boolValue {
AXUIElementSetAttributeValue(
element,
kAXValueAttribute as CFString,
"\(precedingTextValue)\n\(codeBlock)" as CFString
)
return
}
Slide 54
Slide 54 text
No content
Slide 55
Slide 55 text
No content
Slide 56
Slide 56 text
4$4DSFFOTIPU.BOBHFSDBQUVSF*NBHF
func captureScreen(rect: CGRect) async throws -> CGImage? {
let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false)
guard
let mainDisplayID =
NSScreen.main?.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID,
let display = content.displays.first(where: { $0.displayID == mainDisplayID })
else { return nil }
let streamConfig = SCStreamConfiguration()
streamConfig.captureResolution = .automatic
streamConfig.sourceRect = rect
streamConfig.preservesAspectRatio = true
streamConfig.scalesToFit = true
streamConfig.capturesAudio = false
streamConfig.excludesCurrentProcessAudio = true
let filter = SCContentFilter(display: display, excludingWindows: [])
let capturedImage = try await SCScreenshotManager.captureImage(
contentFilter: filter, configuration: streamConfig
)
return capturedImage
}
Slide 57
Slide 57 text
3FUSJFWFBO*NBHF
$(8JOEPX-JTU$SFBUF*NBHF @@@@
func captureScreen(rect: CGRect) -> CGImage? {
let screenShot = CGWindowListCreateImage(
rect,
[.optionOnScreenOnly, .excludeDesktopElements],
kCGNullWindowID,
[.boundsIgnoreFraming, .bestResolution]
)
return screenShot
}
Slide 58
Slide 58 text
(SBOU4DSFFO3FDPSEJOH1FSNJTTJPO
1SJWBDZ4FUUJOHT
Slide 59
Slide 59 text
(SBOU4DSFFO3FDPSEJOH1FSNJTTJPO
$(1SF
fl
JHIU4DSFFO$BQUVSF"DDFTT
r $(1SF
fl
JHIU4DSFFO$BQUVSF"DDFTT
r *OEJDBUFTXIFUIFSTDSFFOSFDPSEJOHJTBMMPXFE
r $(3FRVFTU4DSFFO$BQUVSF"DDFTT
r 1SFTFOUUIFTZTUFNQSPNQU
Slide 60
Slide 60 text
&YUSBDUUIF5FYUGSPNBO*NBHF
ϒϥοΫδϟοΫʹΑΖ͘͠
ࠤ౻लๆ
Slide 61
Slide 61 text
7JTJPO,JU
ϒϥοΫδϟοΫʹΑΖ͘͠
ࠤ౻लๆ
Slide 62
Slide 62 text
&YUSBDUUIF5FYUGSPNBO*NBHF
*NBHF"OBMZ[FS
let analyzer = ImageAnalyzer()
let analysis = try await analyzer.analyze(
image,
orientation: .up,
configuration: configuration
)
let transcript = analysis.transcript
Slide 63
Slide 63 text
&YUSBDUUIF5FYUGSPNBO*NBHF
*NBHF"OBMZ[FS
let analyzer = ImageAnalyzer()
let analysis = try await analyzer.analyze(
image,
orientation: .up,
configuration: configuration
)
let transcript = analysis.transcript
Slide 64
Slide 64 text
No content
Slide 65
Slide 65 text
8SBQ6Q
r 8JOEPX$IBU"TTJTUBOU
r #BTJD6TBHFPG"DDFTTJCJMJUZ"1*
r *OMJOF$IBU"TTJTUBOU
r /PUJ
fi
DBUJPOT
r .PEJGZ$POUFOU
r $PNJD5SBOTMBUPS
r 4DSFFO$BQUVSF"1*BOE0$3
r "96*&MFNFOU*OTQFDUPS
r *OTQFDU"UUSJCVUF/BNFPG"96*&MFNFOU
Slide 66
Slide 66 text
4BNQMF$PEF
➡IUUQTHJUIVCDPNLJTIJLBXBLBUTVNJUSZTXJGU
Slide 67
Slide 67 text
3FGFSFODFT
r 4BNQMF$PEF
IUUQTHJUIVCDPNLJTIJLBXBLBUTVNJUSZTXJGU
r "DDFTTJOHUFYUWBMVFGSPNBOZ4ZTUFNXJEF"QQMJDBUJPOWJB"DDFTTJCJMJUZ "1*
IUUQTNBDEFWFMPQFSTXPSEQSFTTDPNBDDFTTJOHUFYUWBMVFGSPNBOZTZTUFNXJEFBQQMJDBUJPOWJB
BDDFTTJCJMJUZBQJ
r )PXUPHFUTFMFDUFEUFYUBOEJUTDPPSEJOBUFTGSPNBOZTZTUFNXJEFBQQMJDBUJPOVTJOH
"DDFTTJCJMJUZ "1*
IUUQTNBDEFWFMPQFSTXPSEQSFTTDPNIPXUPHFUTFMFDUFEUFYUBOEJUTDPPSEJOBUFTGSPNBOZTZTUFNXJEF
BQQMJDBUJPOVTJOHBDDFTTJCJMJUZBQJ
r 8IZBSFOUUIFNPTUVTFGVM.BDBQQTPOUIF"QQ4UPSF
IUUQTBMJOQBOBJUJVDPNCMPHBQQTPVUTJEFBQQTUPSF
r "MMBCPVUNBD04FWFOUPCTFSWBUJPO
IUUQTEPDTHPPHMFDPNQSFTFOUBUJPOEO&BJ16EVIWKLTS%735D+B&6-C488IU7E()'@946IUNMQSFTFOU