
    ia                        d dl Z d dlZd dlZd dlZd dlmZmZmZm	Z	 d dl
Z
d dlmZmZ d dlZd dlmZmZmZmZ d dlmZ d dlZd dlZ ej0                  e      Zd dlmZmZmZmZmZ d dl m!Z!m"Z" d dlZd dl#Z#d dlmZmZmZmZm$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 d d	l4m5Z5m6Z6m7Z7 d d
l8m9Z9 d dl:m;Z< d dlm=Z=m>Z> d dlm?Z? d dl@mAZA d ZB ej0                  e      Z G d d      ZCy)    N)CallableDictListOptional)Image	ImageDraw)AXUIElementPerformActionAXUIElementSetAttributeValuekAXPressActionkAXValueAttribute)NSString)AXValueGetTypekAXValueCGPointTypekAXValueTypeCGSizeAXValueGetValuekAXValueCGSizeType)CGPointCGSize)r	   r
   r   r   AXUIElementCopyActionNamesAXUIElementCopyAttributeValueAXUIElementCopyAttributeNamesAXUIElementCreateApplicationkAXChildrenAttributekAXDescriptionAttributekAXErrorAPIDisabledkAXErrorAttributeUnsupportedkAXErrorCannotCompletekAXErrorFailurekAXErrorIllegalArgumentkAXErrorSuccesskAXMainWindowAttributekAXRoleAttributekAXTitleAttributekAXWindowsAttribute)CFRunLoopAddSourceCFRunLoopGetCurrentkCFRunLoopDefaultMode)MacElementNode)NSArrayNSMutableArray)	ImageFont)NSScreenc                 n    t        | t        t        f      r| D cg c]  }t        |       c}S | S c c}w )z9Convert NSArray/NSMutableArray to Python list recursively)
isinstancer)   r*   convert_nsarray)valueitems     >/Users/mibo/.openclaw/workspace/data/TuriX-CUA/src/mac/tree.pyr/   r/   2   s5    %'>23278%$%%88L 9s   2c                   R   e Zd Zd ZdedefdZdddedefdZ	ddde
e   fd	Zdddeeef   fd
Zdddede
e   defdZddddedee   dedee   f
dZd Zd Zdej(                  fdZdee   fdZdee   fdZdedeej(                     fdZdefdZddee   dee   fdZy)MacUITreeBuilderc                     d| _         i | _        i | _        t               | _        d | _        d| _        d| _        d | _        d | _	        d | _
        d| _        h d| _        h d| _        y )Nr         >   AXPressAXRaiseAXCancel	AXConfirm
AXSetValue
AXShowMenuAXDecrementAXIncrement>   AXScrollUpByPageAXScrollDownByPageAXScrollLeftByPageAXScrollRightByPage)highlight_index_element_cache
_observersset_processed_elements_current_app_pid	max_depthmax_children_screenshot_annotated_screenshot
app_windowwindow_countINTERACTIVE_ACTIONSSCROLL_ACTIONSselfs    r2   __init__zMacUITreeBuilder.__init__<   sm      #&5  $%)"	$
 
    pidreturnc                      y)z/Setup accessibility observer for an applicationT )rS   rV   s     r2   _setup_observerz MacUITreeBuilder._setup_observer]   s    rU   elementAXUIElement	attributec                 |    	 t        ||d      \  }}|t        k(  r|S |t        k(  ryy# t        $ r
}Y d}~yd}~ww xY w)z:Safely get an accessibility attribute with error reportingN)r   r    r   	Exception)rS   r[   r]   error	value_refes         r2   _get_attributezMacUITreeBuilder._get_attributea   sM    	<WiQUVE9'  66  		s   ( 	( 	;;c                     	 t        |d      \  }}|t        k(  r|rt        |      S g S # t        $ r$}t        j                  d|        g cY d}~S d}~ww xY w)z?Get available actions for an element with proper error handlingNzError getting actions: )r   r    listr_   loggerdebug)rS   r[   r`   actionsrb   s        r2   _get_actionszMacUITreeBuilder._get_actionsq   sY    	7FNE7'GG}$I 	LL21#67I	s   $) ) 	AAAAc                     i }	 t        |d      \  }}|t        k(  r'|r%t        |      D ]  }| j                  ||      ||<    |S # t        $ r#}t
        j                  d|        Y d}~|S d}~ww xY w)z
        Retrieve all available attributes from an accessibility element.
        This uses AXUIElementCopyAttributeNames to get a list of all supported attributes,
        then fetches each attribute's value.
        Nz!Error retrieving all attributes: )r   r    re   rc   r_   rf   rg   )rS   r[   
attributesr`   attribute_namesattrrb   s          r2   _get_all_attributesz$MacUITreeBuilder._get_all_attributes}   s     
	B%B7D%Q"E?'O 1D'+':':7D'IJt$ 2   	BLL<QC@AA	Bs   ?A 	A1A,,A1rolerh   c                      |syt         fd|D              }t         fd|D              }d|v r"|dk(  r j                  |d      }t        |      S d|v r!|dv r j                  |d      }t        |      S |xs |S )	zKDetermine if an element is truly interactive based on its role and actions.Fc              3   :   K   | ]  }|j                   v   y wN)rP   .0actionrS   s     r2   	<genexpr>z3MacUITreeBuilder._is_interactive.<locals>.<genexpr>   s     WwVf(@(@@w   c              3   :   K   | ]  }|j                   v   y wrr   )rQ   rs   s     r2   rv   z3MacUITreeBuilder._is_interactive.<locals>.<genexpr>   s     MW64#6#66Wrw   r<   AXTextField	AXEnabledr8   )AXButtonAXLink)anyrc   bool)rS   r[   ro   rh   has_interactive
has_scrollenableds   `      r2   _is_interactivez MacUITreeBuilder._is_interactive   s     WwWWMWMM
 7"t}'<))';?G=  D,B$B))';?G= ,*,rU   Nparentdepthc           	      v	   K   t        |      }| j                  v ry j                  j                  |       	  j                  |t              }|sy j                  |      }t        ||i dd||      }||_        |r||j                  d<    j                  |t              }	 j                  |t              }
 j                  |t              } j                  |d      } j                  |d      }|	r|	|j                  d<   |
r|
|j                  d	<   |r||j                  d
<   |(t        |      |_        t        |      |j                  d<   |r||j                  d<    j                   j                  |d            } j                   j                  |d            }t!        d |D              }t!        d |D              }|\  }}|\  }}|d|z  z   }|d|z  z   }|dz  }|dz  }||f}||f}|j                  j#                  ||d        fd} ||      rd|_        nd|_         j'                  |||      |_        g d}d}|j                  j+                  dd      \  }}|j                  j+                  dd      \  }}||z   }||z   }|dkD  rK|dkD  rF|dk  rA|dk  r<|dk\  r7|dk\  r2|dk  r-|dk  r(|D ]#  }||j                  v s|j                  |   "d}% |ri|j(                  rD|dvr@ j,                  |_        | j.                   j,                  <    xj,                  dz  c_        nd|_        | j.                  d| <    j                  |t0              }|r| j2                  k  r	 t5        t7        |            }| j8                  kD  r)t:        j=                  d j8                   d | d!| d"       t?        |      }|D ]?  } jA                  ||||dz          d{   } | s%|jB                  jE                  |        A 	 |S |r8| j2                  k\  r)t:        j=                  d$ j2                   d%| d&| d'       |S 7 c# tF        $ r#}!t:        jI                  d#|!        Y d}!~!|S d}!~!ww xY w# tF        $ r+}!t:        j=                  d(t        |!              Y d}!~!yd}!~!ww xY ww))zProcess a single UI elementNTF)ro   
identifierrk   
is_visible	on_screenr   app_pidrh   rz   	AXSubroletitler0   descriptionr   subrole
AXPositionAXSizec              3   4   K   | ]  }t        |d         yw   Nroundrt   vs     r2   rv   z4MacUITreeBuilder._process_element.<locals>.<genexpr>   s     :'QU1a['   c              3   4   K   | ]  }t        |d         ywr   r   r   s     r2   rv   z4MacUITreeBuilder._process_element.<locals>.<genexpr>   s     ;(QU1a[(r   g?g?positionsizec                    j                   syj                  j                  \  }}j                   d   \  }}j                   d   \  }}||||z   ||z   f}| j                  j	                  d      }| j                  j	                  d      }	|r|	sy|\  }
}|	\  }}|
|z   }||z   }|dk  xs |
|k\  xs |dk  xs ||k\   }|sy|r0|\  }}}}|
|dz
  k\  xr ||dz   k  xr ||dz
  k\  xr ||dz   k  }|syy)NFr   r   r      T)rN   rL   r   rk   get)r[   screen_widthscreen_heightwxwywwwhwindow_boundsposr   xywh
elem_rightelem_bottomr   wx1wy1wx2wy2window_overlaprS   s                         r2   r   z5MacUITreeBuilder._process_element.<locals>.is_visible   sX    .2.>.>.C.C+m4B0B!#Rb"r' :((,,Z8))--f5$ 11U
!e !O '%'1$' &		 !  !)6&Cc3SU
 -"c!e+-SU
- $s1u,	 # *$rU   )r   r0   r   r   r   r   r   r   r   r   g{Gzt?g?r      )AXGroupAXImageAXSplitGroupAXScrollAreactx_zMax children limit (z) exceeded for element z. Found z/ children. Some elements will not be processed.zError processing children: zMax depth limit (z) reached for element z. Children at depth z will not be processed.zError processing element: )%strrH   addrc   r"   ri   r(   _elementrk   r#   r   r   r~   r   _convert_axvalue_to_point_convert_axvalue_to_sizetupleupdater   r   is_interactiver   rD   rE   r   rJ   lenre   rK   rf   r`   r/   _process_elementchildrenappendr_   warning)"rS   r[   rV   r   r   element_identifierro   rh   noder   r0   r   
is_enabledr   raw_posraw_sizer   r   x0y0r   r   r   important_attrs
should_addx1y1rm   children_refchildren_countchildren_listchild
child_noderb   s"   `                                 r2   r   z!MacUITreeBuilder._process_element   s     \!9!99  $$%78V	&&w0@AD''0G "-D $DM -4	*''1BCE''1BCE--g7NOK,,WkBJ))';?G+0(+0(1<.%"&z"2-1*-=	*-4	*55d6I6I'S_6`aG44T5H5HRZ5[\H :'::H;(;;DFBDAqc!eBc!eBCACABxHq6DOO""$$ 
)V $!%!&"&"6"6wg"ND\OJOO''
F;EBr//%%ff5CAaaBaB5yQY1c6a#g"'bTUgZ\`aZafhlmfm+Dt.4??43H3T%)
 , &&47m+m+/+?+?D(@DD''(<(<=((A-(+/D(GKD''$/A.B(CD  ..w8LML 6F%(l);%<N%(9(99';D<M<M;NNefjekks  uC  tD  Ds  &t  u$3L$AM!.+/+@+@TSX[\S\+]%]
% MM00< "/ K %4>>"900@@VW[V\\pqvpw  xO  P  QK &^ ! FNN%@#DEE KF  	LL5c!fX>?	s   6R9R R9I<R R  BR 7A6Q -Q.Q 5Q R R9;R R9Q 	Q?Q:4R 9R9:Q??R 	R6!R1,R91R66R9c                     | j                   j                          | j                  j                          d| _        d| _        ddl}|j                          t        j                  d       y)z'Cleanup observers and release resourcesr   Nz;MacUITreeBuilder cleanup completed: all references released)	rE   clearrH   rD   rI   gccollectrf   rg   )rS   r   s     r2   cleanupzMacUITreeBuilder.cleanupC  sU     	!!#  &&(  $ 	


 	RSrU   c                     d| _         | j                  j                          | j                  j                          t        j                  d       y)z#Reset the state between major stepsr   zMacUITreeBuilder state resetN)rD   rE   r   rH   rf   rg   rR   s    r2   reset_statezMacUITreeBuilder.reset_stateU  s=     !!#  &&(
 	34rU   c                 n   	 t         j                  d       t        j                         }|j                  \  }}t        ||      }d}|dk\  rd}n|dkD  rd}|dkD  rt        d||z        t        d||z        f}t        t        d      rt        j                  j                  }nt        j                  }|j                  ||      }t         j                  d	|||j                  |j                  |       || _        | j                  S # t        $ r"}t         j                  d
|        Y d}~yd}~ww xY w)z*Capture a screenshot of the current screenzPCapturing screenshot............................................................r   i      i     
Resampling)resamplez<Downscaled screenshot from %sx%s to %sx%s (scale factor: %s)zFailed to capture screenshot: N)rf   rg   	pyautogui
screenshotr   maxhasattrr   r   LANCZOSresizewidthheightrL   r_   r`   )	rS   r   r   r   max_dimscale_factortarget_sizer   rb   s	            r2   capture_screenshotz#MacUITreeBuilder.capture_screenshot`  s$   "	LLkl"--/J&OOME6%(GL
 $ 4 a"1e|&;<c!V|E[>\]5,/$//77H$}}H'..{X.N
R$$%%   *D### 	LL9!=>	s   DD	 		D4D//D4c                 n   |rt        |      dk(  rt        |      }d}t        j                  ||      }|rht	        |j                  d            t        j                         d   z  }t	        |j                  d            t        j                         d   z  }||fS t        j                  d|        y)z0Convert AXValue (CGPoint) to Python tuple (x, y)r   z'x:([-+]?\d+\.?\d*)\s+y:([-+]?\d+\.?\d*)r   r   Failed to match pattern: N
r   r   researchfloatgroupr   r   rf   rg   )rS   axvaluepatternmatchr   r   s         r2   r   z*MacUITreeBuilder._convert_axvalue_to_point  s    ~g.!3'lG@GIIgw/E%++a.)INN,<Q,??%++a.)INN,<Q,??1v8	BCrU   c                 n   |rt        |      dk(  rt        |      }d}t        j                  ||      }|rht	        |j                  d            t        j                         d   z  }t	        |j                  d            t        j                         d   z  }||fS t        j                  d|        y)z8Convert AXValue (CGSize) to Python tuple (width, height)r   zw:([\d.]+)\s+h:([\d.]+)r   r   r   Nr   )rS   r   r   r   r   r   s         r2   r   z)MacUITreeBuilder._convert_axvalue_to_size  s    ~g.!3'lG0GIIgw/Eekk!n-	0@0CCu{{1~.1A!1DDv&8	BCrU   rootc                   	
 | j                   st        j                  d       y | j                   j                         }t	        j
                  |      	| j                   j                  \  d}t        j                         j                  |      
g dd 	 | j                  dk(  rD| j                  d   \  }}|z  }|z  }| j                  d   \  }}|z  }|z  }||||z   ||z   f	
fd
|r*t        j                  d|j                           |       || _        |S #  }}dd||ft        j                  d	       Y cxY w)Nz$No screenshot available to annotate.   )r   )redbluegreenyellowpurpler   r   r   r   zError getting window boundsc                    	 | j                   r | j                  | j                  d   \  }}|z  }|z  }| j                  d   \  }}|z  }|z  }\  }}}}t        ||z   |      }	t	        ||      }t        ||z   |      }
t	        ||      }t        | j                        }|t              z     }j                  |||	|
gd|       t        |      }j                  d|      }|d   |d   z
  }j                  ||f||       | j                  D ]
  } |        y # t        $ r
}Y d }~(d }~ww xY w)	Nr   r   r   )r   outliner   )fontr   )fillr  )r   rD   rk   minr   intr   	rectangler   textbboxtextr_   r   )r[   r   r   r   r   r   r   r   r   rightbottomnumbercolorr	  	text_bboxtext_heightrb   r   color_palettedrawr  process_elementr   r   r   s                     r2   r  z=MacUITreeBuilder.annotate_screenshot.<locals>.process_element  sc   $$)@)@)L"--j9DAq%A&A"--f5DAq%A&A)6&Cc3AsOEAc
A Q_FAsA !8!89F)&3}3E*EFENNAq%#85NQv;D $fd FI"+A,1"=KIIq!fdTIB
 !))& *	  s   DD* *	D=8D=zStarting annotation from root: )rL   rf   rg   copyr   Drawr   r+   load_defaultfont_variantrO   rN   r`   ro   rM   )rS   r   	annotated	font_sizer   r   r   r   r  r  r  r  r   r   r   s           @@@@@@@r2   annotate_screenshotz$MacUITreeBuilder.annotate_screenshot  s\   LL?@$$))+	~~i(&*&6&6&;&;#m	%%'44)4DD	8  A%4Bl"m#0Bl"m#!#Rb"r' :	' 	'< LL:499+FGD!%."O	8!=B2rNMLL67s   AD2 2!Ec                 8    | j                   syd| j                   iS )z'Get both UI tree and vision informationNr   )rM   rR   s    r2   get_vision_contextz#MacUITreeBuilder.get_vision_context  s%    )) $44
 	
rU   c                 	  K   	 | j                   j                          | j                  j                          d| _        |,| j                   t
        j                  d       t        d      ||| _        ddl}	 |j                  ddt        | j                        gdd      }|j                  dk7  r;t
        j                  d	| j                   d
       d| _        | j                          y	 | j                  | j                        st
        j!                  d       yt
        j                  d| j                          t#        | j                        }t
        j                  d       t%        |t&        d      \  }}|t(        k(  rt
        j                  d| d| d       nit
        j                  d|        |t*        k(  rt
        j                  d       y|dk(  r,t
        j                  d       d| _        | j                          yt-        dt        |      i d| j                  d      }||_        t
        j                  d       t%        |t0        d      \  }}	|dk(  ry|t(        k7  s|	st
        j!                  d| d       t%        |t2        d      \  }}
|t(        k(  rD|
rB	 t5        |
      }|r|d   }	t
        j                  d|	        nt
        j!                  d       nt
        j                  d | d       |	rt
        j                  d!|	        | j7                  |	| j                  |       d{   }|r|j8                  j;                  |       |j<                  j?                  d"      }|j<                  j?                  d#      }|r|r||d$| _         d%| _!        |S t
        j                  d&       |S # t        $ r#}t
        j                  d|        Y d}~d}~ww xY w# t        $ r#}t
        j                  d|        Y d}~d}~ww xY w7 # t        $ rL}dt        |      vr5t
        j                  d't        |              ddl"}|jG                          Y d}~yd}~ww xY ww)(z(Build UI tree for a specific applicationr   Nz9No app is currently open - waiting for app to be launchedzNo app is currently openpsz-pT)capture_outputr	  zProcess with PID z is no longer runningzError checking process status: z&Failed to setup accessibility observerzCreating AX element for pid z+Testing accessibility permissions (Role)...z"Successfully got role attribute: (z, )zError getting role attribute: zBAccessibility is not enabled. Please enable it in System Settings.izYError -25204: Accessibility connection failed. The app may have been closed or restarted.application)ro   r   rk   r   r   r   z Trying to get the main window...z-25212)NzWindow not foundz"Could not get main window (error: z&), trying fallback attribute AXWindowsz0Fallback: selected first window from AXWindows: z*Fallback: AXWindows returned an empty listz"Failed to iterate over AXWindows: z1Fallback failed: could not get AXWindows (error: zFound main window: r   r   r   r   z6Could not determine a main window for the application.zError building tree: )$rH   r   rE   rD   rI   rf   rg   
ValueError
subprocessrunr   
returncoder`   r   r_   rZ   r   r   r   r"   r    r   r(   r   r!   r$   re   r   r   r   rk   r   rN   rO   	traceback	print_exc)rS   rV   r"  resultrb   app_refr`   	role_attrr   main_window_refwindowswindows_listwindow_nodemain_pos	main_sizer%  s                   r2   
build_treezMacUITreeBuilder.build_tree  s     g	$$**,%%'#$D {t44<XY !;<<(+% D#tS9N9N5O(Paelpq$$)LL#4T5J5J4KK`!ab,0D)LLN	 * ''(=(=>GHLL78M8M7NOP243H3HIGLLFG<WFVX\]E9'A%9+UVWX=eWEF//LL!ef  f_LL#|~,0D)LLN!"w<--D $DMLL;<%B7Lbdh%i"E? /'!CE7Jpqr!>wH[]a!bwO+O'+G}'.:1oO"LL+[\k[l)mn"NN+WX LL#TUZT[[\!]^2?2CDE$($9$9/4K`K`bf$ggMM((5&1155jA'2266v>		$, )'DO )*D% K UVKS  D>qcBCCDf % O'I!%MNNO h"  	)Q74SVH=> ##%	s   R A6P( ;A3O .R /1P(  R !B<P( R 1P( R AP( )R *AP( .A O7 .AP( P&A-P( 0R 1P( R 	O4O/)P( /O44P( 7	P# PP( P##P( (	Q=1AQ83R 8Q==R )Nr   rr   )__name__
__module____qualname__rT   r  r~   rZ   r   r}   rc   r   ri   r   rn   r   r   r(   r   r   r   r   r   r   r   r   r  dictr  r0  rY   rU   r2   r4   r4   ;   sZ   
B3 4 m    
M 
d3i 
= T#s(^  -} -C -$s) -X\ -*_m _# _xXfOg _wz _  DL  M[  D\ _BT$	5$EKK $LHUO  8E? = =8EKK;P =~
D 
iHSM iXn=U irU   r4   )Dasyncior   numpynploggingtypingr   r   r   r   r   PILr   r   CocoaApplicationServicesr	   r
   r   r   
Foundationr   time
HIServices	getLoggerr1  rf   r   r   r   r   r   Quartzr   r   objcr   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   CoreFoundationr%   r&   r'   src.mac.elementr(   Quartz.CoreGraphicsCoreGraphicsCGr)   r*   r+   AppKitr,   r/   r4   rY   rU   r2   <module>rI     s     	   1 1     y y   			8	$ | | "       , Z Y *   .   
		8	$\ \rU   