
    i                       d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
Z
d dlmZmZmZmZmZmZmZ d dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d dlmZ d d
lmZmZ d dlmZ d dl m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+m,Z,m-Z- d dl.m/Z/m0Z0 d dl1m2Z2 d dl3m4Z4m5Z5m6Z6m7Z7 d dl8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z? d dl@mAZA d dlBmCZC d dlDmEZEmFZFmGZGmHZH d dlImJZJ d dlKmLZL d dlMmNZN d dlOmPZP d dlQmRZR d dlS  e         ej                  eU      ZV ede/       ZWd!ZXeXfd*d"ZYd+d#ZZd$ Z[d,d%Z\d-d&Z]d.d'Z^ G d( d)      Z_y)/    )annotationsN)Path)AnyCallableDictListOptionalTypeTypeVar)datetime)load_dotenv)BaseChatModel)r
   )OrderedDict)
ChatOpenAIAzureChatOpenAI)ChatAnthropic)ChatGoogleGenerativeAI)
ChatOllama)BaseMessage)observe)RateLimitError)Image	ImageDraw	ImageFont)	BaseModelValidationError)MessageManager)BrainPrompt_turixActorPrompt_turixMemoryPromptPlannerPrompt)ActionResult
AgentErrorAgentHistoryAgentHistoryListAgentOutputAgentStepInfo
AgentBrain)RecordStore)BrainSearchFlow)load_skill_metadataload_skill_contentsformat_skill_catalogformat_skill_context)Planner)
Controller)MacUITreeBuilder)time_execution_async)OutputSchemas)*T)bound<   c                    | j                         j                         } t        j                  dd|       } | j                  d      } | sd} | d | S )Nz
[^a-z0-9]+-task)striplowerresub)r:   max_lens     C/Users/mibo/.openclaw/workspace/data/TuriX-CUA/src/agent/service.py_task_to_slugrA   D   sI    ::<D66-d+D::c?D>    c                H    |j                  d      }t        |       }| d| S )Nz%Y-%m-%d_)strftimerA   )r:   nowdate_strslugs       r@   _default_agent_idrI   L   s+    ||J'HDZqrB   c                    t        j                         }| j                  |d       t        j                  |j                               j                  d      }d| S )NPNGformatutf-8zdata:image/png;base64,)ioBytesIOsavebase64	b64encodegetvaluedecode)
screenshotimg_byte_arrbase64_encodeds      r@   screenshot_to_dataurlrY   Q   sO    ::<LOOLO/%%l&;&;&=>EEgNN#N#344rB   c           	         | y| j                   j                  t        | dd      t        | dd      t        | dd      t        | dd      g}dj                  d |D              S )N 
model_namemodelopenai_api_basebase_url c              3  T   K   | ]   }|st        |      j                          " y wN)strr<   ).0parts     r@   	<genexpr>z%_llm_identity_text.<locals>.<genexpr>a   s     @%$4CIOO%%s   (()	__class____name__getattrjoin)llmpartss     r@   _llm_identity_textrm   W   sf    
{\2&Wb!&+Z$E 88@%@@@rB   c                    | yt        | dd       }|t        |      S t        | t        t        f      r"t        |       d}t        fd|D               S y)NF_turix_supports_response_format)deepseekminimaxzm2.5moonshotkimic              3  &   K   | ]  }|v  
 y wrb    )rd   tokenidentitys     r@   rf   z/llm_supports_response_format.<locals>.<genexpr>n   s     I6HUu(6Hs   T)ri   bool
isinstancer   r   rm   any)rk   explicitunsupported_tokensrw   s      @r@   llm_supports_response_formatr}   d   sa    
{s=tDHH~#
O45%c*PI6HIIIIrB   c                   t         t        f}t        t        f}t        f}t        | |      rKt        |       s.t        j                  dt        | dt        | dd                   | S |}t        t        | dd      xs t        | dd      xs d      j                         }| xs d|v }|rt        |t              r|j                  d	      }	|j                  d
      }
|	d
k(  rt        |
t              rud	d
i}|
j                  d      r|
j                  d      |d<   |
j                  d      |
j                  d      |d<   |
j                  d      |
j                  d      |d<   |}| j                  |      S t        | |      r| j                  |      S t        | |      rZd}t        |t              r2|j                  d
      }
t        |
t              r|
j                  d      }| j                  |xs d      S | S )u  
    Wrap *any* LangChain chat model with the right structured-output mechanism:

    • ChatOpenAI / AzureChatOpenAI  → bind(response_format=…)      (OpenAI style)
    • ChatAnthropic / ChatGoogleGenerativeAI → with_structured_output(…) (Claude/Gemini style)
    • ChatOllama → bind(format=<json schema>) (Ollama json schema)
    • anything else → returned unchanged
    zXStructured response_format is disabled for model '%s'; falling back to prompt-only JSON.r\   r]   unknownr^   r[   r_   zapi.openai.comtypejson_schemanameschemaNstrict)response_formatjsonrL   )r   r   r   r   r   ry   r}   loggerinfori   rc   r<   dictgetbindwith_structured_output)rk   SchemaStructured_OutputOPENAI_CLASSESANTHROPIC_OR_GEMINIOLLAMA_CLASSESr   r_   is_openai_cloudschema_typer   flatr   s                r@   to_structuredr   r   s    8B?6SN< 8BmN#~&+C0KKj\73+KL J !ws$5r:`gc:WY>Z`^`aggi'<J-=-Iz&$7 **V,K **]3Km+
;0M.??6*#.??6#:DL??8,8%0__X%>DN??8,8%0__X%>DN"&xxx88#*+))*;<<#~&fd# **]3K+t,$2xxv/x00 JrB   c                  ~   e Zd Z e       dddddddddddddddddd	dg d
ddddddf	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d6dZd7dZd8dZd7dZd9dZd7dZ	d:dZ
d;dZ	 d<	 	 	 	 	 	 	 	 	 d=dZd7dZd<d>dZd<d>dZd7dZd7dZd7dZd7dZ ed      d?d       Z ed      d<d@d        ZdAd!Z	 	 	 	 	 	 	 	 dBd"Z ed#      dCd$       ZdDd%Z	 	 	 	 	 	 	 	 dEd&Z	 	 	 	 	 	 	 	 dEd'ZdFd(ZdGd)Zd7d*ZdHdId+Z d, Z!d-Z"d.Z#d<dJd/Z$dKd0Z%dLd1Z&dLd2Z'd<dMd3Z(d<dNd4Z)d5 Z*y)OAgentFTNi  rN        
   i }  )
titler   r   roletabindexz
aria-labelplaceholdervaluealtzaria-expandedi  autoc                    d| _         t        j                         | _        |xs t	        || j                        | _        || _        |r't        |      j                         j                         nd | _
        | j                  rHt        | j                  dz  | j
                  z        | _        t        | j                  dz        | _        nMd| _        t        j                  j!                  t        j                  j#                  t$              d      | _        || _        ||nt)        d|dz        | _        || _        || _        t1        |t2        j4                  t6              | _        t1        |t2        j:                  t<              | _        t1        |t2        j@                  tB              | _"        || _#        t1        |t2        jH                  tJ              | _&        || _'        || _(        || _)        || _*        || _+        |xs d| _,        || _-        || _.        d | _/        || _0        || _1        || _2        || _3        |	rt        |	      j                         nd | _4        t)        d|
xs d      | _5        g | _6        g | _7        d| _8        d| _9        d| _:        tw               | _<        || _=        || _>        d | _?        t               | _A        d | _B        t               | _C        d	| _D        | j                          | j.                  r|st        d
      t        j                  j!                  | j                  | j
                         | _        t        j                  j!                  | j                  d      | _G        t        | j                  | jT                  xs d      | _I        t        j                  j!                  | j                  d      | _J        t        | j                  | jT                  xs d      | _K        t        | j                        | _M        t        j                  d| j
                          t        j                  d| j                          | jf                  r| jh                  rt        | jh                        | _6        | jl                  s!t        j                  d| jh                         nydj!                  d | jl                  D              } t        j                  dt        | jl                        | jh                  |        n!| jf                  rt        j                  d       | jL                  rd}!| jf                  r!| jl                  rt        | jl                        }!| jd                  s| jf                  r| jF                  nd }"t        | jL                  | j                  | j`                  |"| jd                  |!| jf                  | jl                  | jj                  | jV                  | jX                        | _T        n!| jf                  rt        j                  d       | j                          d | _V        || _W        || _X        t        g       | _Z        d| _[        d| _\        || _]        || _^        d| __        d| _`        d| _a        d| _b        d| _c        d| _d        g | _e        g | _f        d | _g        d| _h        y )NFimages
temp_files      rN   r   r[   successz)Agent ID is required for resuming a task.recordsencodingmemory_snapshotsz
Agent ID: zAgent memory path: zNo skills loaded from %s, c              3  4   K   | ]  }|j                     y wrb   )r   )rd   skills     r@   rf   z!Agent.__init__.<locals>.<genexpr>+  s     'V@Uu

@Us   zLoaded %d skill(s) from %s: %sz0Skills enabled but no skills directory provided.)planner_llmr:   max_input_tokenspreplan_llm
use_searchskill_catalog
use_skillsavailable_skillsskills_max_charssave_planner_conversation_path'save_planner_conversation_path_encodingzQSkills enabled but planner is disabled. Set agent.use_plan=true to select skills.)history)iwait_this_stepr   rF   current_timerI   agent_idr:   r   
expanduserresolveartifacts_dirrc   
images_dirsave_temp_file_pathospathrj   dirname__file__memory_budgetmaxsummary_memory_budgetoriginal_taskresumer   r3   MEMORY_RESPONSE_FORMATMemoryOutput
memory_llmBRAIN_RESPONSE_FORMATBrainOutput	brain_llmACTION_RESPONSE_FORMATActorOutput	actor_llmplanner_llm_rawPLANNER_RESPONSE_FORMATPlannerOutputr   save_actor_conversation_path%save_actor_conversation_path_encodingsave_brain_conversation_path%save_brain_conversation_path_encodingr   r   include_attributesmax_error_lengthscreenshot_annotatedr   use_uir   r   
skills_dirr   r   selected_skillsskill_context	next_goalbrain_thoughtr1   mac_tree_builder
controllermax_actions_per_steplast_step_actionr   goal_action_memory	last_goalbrain_contextstatus_setup_action_models
ValueError
record_dirr)   record_storememory_snapshot_dirmemory_snapshot_storer*   brain_searchr   r   r+   lenr-   r/   plannerinitiate_messages_last_resultregister_new_step_callbackregister_done_callbackr%   r   n_stepsconsecutive_failuresmax_failuresretry_delay_paused_stoppedbrain_memorysummary_memoryrecent_memorypending_recent_memorymemory_snapshot_filesinfor_memorylast_pidask_for_help)#selfr:   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   tool_calling_methodr   skill_namesr   planner_preplan_llms#                                      r@   __init__zAgent.__init__   sd   \ $$LLN N$5dD<M<M$N	KXT-0;;=EEG^b!$"4"4x"?$--"OPDO'*4+=+=+L'MD$&DO')ww||BGGOOH4M|'\D$*>S>_%:ehijly|}l}e~"!'
M4X4XZfg&y-2U2UWbc&y-2V2VXcd*(m6[6[]jk,H)5Z2,H)5Z2.L+7^7ibi4"4 0$(! 0$$;E$z*5574 #A'7'<1 = "! 0 2$$8! $"--(]!!# ;;xHII#%77<<0H0HT]]O#] '',,t'?'?K'OO??J7
 $&77<<0H0HJ\#] %0$$??J7&
" ,D,=,=>j01)$*B*B)CDE??t$7$HD!((6H"ii'V@U@U'VV<c$BWBW>XZ^ZiZikvw__KKJKM4#8#8 4T5J5J K;???doo$"6"6dh" ,,YY!%!6!6/??+??!%!6!6!%!6!6/3/R/R8<8d8dDL __KKkl 	  *D'&<# *:")E$%!(& %'";="!rB   c                $   | j                   j                  j                  | _        t	        | j                   d      r| j                   j
                  | _        y t	        | j                   d      r| j                   j                  | _        y d| _        y )Nr\   r]   Unknown)rk   rg   rh   chat_model_libraryhasattrr\   r]   r  s    r@   _set_model_nameszAgent._set_model_names_  s]    "&(("4"4"="=488\*"hh11DOTXXw'"hhnnDO'DOrB   c                p    |dk(  r1| j                   dk(  ry | j                   dk(  ry| j                   dk(  ryy y )Nr   r   r   function_callingr   )r  )r  r  s     r@   set_tool_calling_methodzAgent.set_tool_calling_methodh  sG    &(&&*BB((L8)((,==) )rB   c                    | j                   j                  j                         | _        t	        j
                  | j                        | _        y)z6Setup dynamic action models from controller's registryN)r   registrycreate_action_modelActionModelr&   type_with_custom_actionsr  s    r@   r   zAgent._setup_action_modelss  s6    ??33GGI&??@P@PQrB   c                    | j                   }| j                  r*| j                  D ]  }|j                  s|j                  } |S rb   )r  r   current_app_pid)r  
latest_pidrs      r@   get_last_pidzAgent.get_last_pidx  s@    ]]
&&$$!"!2!2J ' rB   c                L   g }| j                   r|j                  d| j                   z          | j                  r|j                  d| j                  z          | j                  r|j                  d| j                  z          dj	                  |      j                         | _        y )NzSummarized memory:
zRecent steps:
zPending steps:


)r   appendr  r  rj   r;   r   )r  rl   s     r@   _refresh_brain_memoryzAgent._refresh_brain_memory  s    LL/$2E2EEFLL*T-?-??@%%LL+d.H.HHI"KK.446rB   c                ~   t        |dd       }t        |t              r|S t        |t              r|S t        t        |d|            }t	        j
                  dd|j                               }t	        j
                  dd|      j                         }t        j                  d|        t        j                  |      S )Nparsedcontentz^```(json)?r[   z```$z[Memory] Raw text: )ri   ry   r   rc   r=   r>   r;   r   debugr   loads)r  responser$  memory_textcleaned_memory_responses        r@   _extract_memory_payloadzAgent._extract_memory_payload  s    8T2fd#Mh%O'(Ix@A"$&&[=N=N=P"Q"$&&"6M"N"T"T"V*+B*CDEzz122rB   c                
  K   d| d| dg}| j                   j                          | j                   j                          | j                   j                  |       | j                   j	                         }| j
                  j                  |       d {   }| j                  |      }t        |j                  dd            j                         }t        |j                  dd            j                         }||fS 7 kw)Ntextr   r   r%  summaryr[   	file_name)memory_message_manager_remove_last_state_message_remove_last_AIntool_messageadd_state_messageget_messagesr   ainvoker+  rc   r   r;   )	r  r)  context_labelmemory_contentmemory_messagesr(  r$  r/  r0  s	            r@   _run_memory_summaryzAgent._run_memory_summary  s      +_D>
 	##>>@##@@B##55nE55BBD00AA--h7fjjB/0668

;34::<		!!	 Bs   BDDA,Dc                    |sy ||n| j                   }|xs	 d| d| d}| j                  j                  |||      }| j                  j	                  |||d       |S )Nmemory_snapshot__step_.txtstep)r0  sourcer@  )r   r   rQ   r  r!  )r  r)  r0  rA  step_override
step_value	safe_name
saved_names           r@   _save_memory_snapshotzAgent._save_memory_snapshot  s|     &3&?]T\\
R#3F86*T!R	//44[)R\4]
""))' "	
 rB   c                @   K   | j                          d{    y7 w)zf
        Summarise recent memory to reduce its size without counting summaries in the budget.
        N)_summarise_recent_memoryr  s    r@   _summarise_memoryzAgent._summarise_memory  s      ++---s   c                  K   | j                   sy 	 | j                  | j                   d       d {   \  }}| j                  | j                   |d|       |s&t        j                  d       | j                          y | j                  r1dj                  | j                  |g      j                         | _        n|| _        d| _         | j                  |       d {    | j                          y 7 # t        $ rG t        j	                  d       | j                  | j                   dd|       | j                          Y y w xY w7 jw)Nz+Summarize the following recent-step memory.z+[Memory] Failed to summarize recent memory.r[   recentrB  z@[Memory] Empty summary from memory model; keeping recent memory.
)r  r:  	Exceptionr   	exceptionrF  r"  warningr   rj   r;   _summarise_summary_memoryr  rB  r/  r0  s       r@   rH  zAgent._summarise_recent_memory  s5    !!		'+'?'?""=( "GY 	""4#5#5y(Zg"hNN]^&&("&))T-@-@',J"K"Q"Q"SD")D,,=,III""$-"  	JK&&t'9'92xWd&e&&(		" 	JsE   EC7 C5C7 B'EE
 E5C7 7AEEEEc                  K   | j                   sy t        | j                         | j                  k  ry 	 | j                  | j                   d       d {   \  }}| j                  | j                   |d|       |s&t
        j                  d       | j                          y || _         | j                          y 7 f# t        $ r7 t
        j                  d       | j                  | j                   dd|       Y y w xY ww)NzJSummarize the following accumulated summaries into a higher-level summary.z3[Memory] Failed to summarize accumulated summaries.r[   r/  rL  z>[Memory] Empty high-level summary; keeping existing summaries.)
r   r   r   r:  rN  r   rO  rF  rP  r"  rR  s       r@   rQ  zAgent._summarise_summary_memory  s     ""t""#t'A'AA	'+'?'?##\( "GY 	""4#6#6	9\i"jNN[\&&(%""$"  	RS&&t':':B	Yf&g	s;   1C?B< B:B< AC?:B< <=C<9C?;C<<C?c                
  K   t        | j                  j                         d      }|sy| j                  |d      d   }|r|d   nd}|d   }d| d| }| j                  j	                         D cg c]  }|j                         s| }}|D cg c]  }|j                  d| d	      r| }}|j                  |       d
j                  |      j                         | _        | j                          yc c}w c c}w w)z'
        Update memory content
        T)reverseNr   current_stater   Step  | Eval: pending | Goal:  |rM  )
sortedr   keysr  
splitlinesr;   
startswithr!  rj   r"  )r  sorted_stepsrV  	step_goalstep_idlinelnpending_liness           r@   _update_memoryzAgent._update_memory  s    
 d00557F**<?;OL2?M+.T	q/ wi8D&*&@&@&K&K&M\&MQSQYQYQ[&M\&3^m2==5QXPYY[I\;]m^T"%)YY}%=%C%C%E"""$	 ]^s+   A0D2C9C9DC>-C>1ADc                H   | j                   sy| j                         | j                  | j                  | j                  | j
                  | j                  | j                  | j                  | j                  | j                  | j                  | j                  d}t        j                  j                  | j                   d      }t        j                  j!                  |      r4t        j"                  t        j                  j!                  |      d      nd t%        |d| j&                        5 }t        j                  j)                  |      dkD  r|j+                  d       |j-                  t/        j0                  |d	d
       dz          ddd       y# 1 sw Y   yxY w)z4
        Save the current memory to a file.
        N)pidr:   r   r   r  r   r@  r   r  r  r   r  memory.jsonlTexist_okwr   r   Fc                <    t        | t              rt        |       S | S rb   )ry   setlist)os    r@   <lambda>z#Agent.save_memory.<locals>.<lambda>%  s    V`abdgVh4PQ7KonoKorB   )ensure_asciidefaultrM  )r   r  r:   r   r   r  r   r   r   r  r  r   r  r   r   rj   r   makedirsopenr   getsizetruncatewriter   dumps)r  datar0  fs       r@   save_memoryzAgent.save_memory  s-    ''$$&II $ 5 5 --!//LL"11%)%?%?!//%)%?%?%)%?%?
 GGLL!9!9\K	BD''//R[B\BGGOOI.>bf)S4+U+UVZ[wwy)A-

1GGDJJt%Aopswwx WVVs   0AFF!c           	       K   | j                   syt        j                  j                  | j                   d      }t        j                  j	                  |      rt        |d| j                        5 }|j                         }ddd       t              dk\  rt        j                  |d         }|j                  dd      | _        |j                  d	d      | _        |j                  d
g       | _        |j                  dt                     | _        | j                   rEt        | j                   j#                         D ci c]  \  }}t%        |      | c}}      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  d| j,                        | _        |j                  dg       | _        | j*                  r| j*                  j1                         D cg c]  }|j3                         s| }}g }	g }
|D ])  }d|v r|
j5                  |       |	j5                  |       + |
rtdj                  | j(                  dj                  |
      fD cg c]  }|s|	 c}      j3                         | _        dj                  |	      j3                         | _        d|vrd|vr| j7                          d{    n| j9                          |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        t@        jC                  d|        yyy# 1 sw Y   xY wc c}}w c c}w c c}w 7 w)z6
        Load the current memory from a file.
        Nrg  r  r   r   r:   r[   rf  r  r   r   r  r  r   r  z| Eval: pending |rM  r   r   r@  zLoaded memory from )"r   r   r   rj   existsrs  r   	readlinesr   r   r'  r   r:   r  r  r   r   itemsintr   r  r  r   r  r\  r;   r!  _rebuild_memory_from_contextr"  r   r   r   r   r   )r  r0  ry  linesrx  kvrb  recent_lineskeep_recentmove_pendings              r@   load_memoryzAgent.load_memory'  s     ''GGLL!9!9>J	77>>)$it/Y/YZ^_ [5zQzz%), HHVR0	 $ 5$(HH^R$@!%)XXo{}%M"%%)4DL^L^LdLdLf5gLfDAqc!faiLf5g)hD&&*hh/?&D#-1XX6Mr-R*%)XXor%B"-1XX6MtOiOi-j*-1XX6Mr-R*%%151C1C1N1N1P#_1P2TVT\T\T^B1PL#_-/K.0L*."4(//3'..r2	 +
 $59YY+/+E+EtyyQ]G^*_f*_BceR*_f6%' 2 .2YY{-C-I-I-K*#4/O44O;;===..0(,1CT(J%!%+r!:#xx21)=>I  %ZZ 6h $` g >s_   A0N 2M!CN M.
B2N M4"M4&A N M9M9AN +M>,A5N !M+&N c                  K   d| _         d| _        d| _        g | _        t	        | j
                  j                               }|s| j                          y |d   }|D ]t  }| j
                  |   j                  di       }|j                  d      }||k(  rJd| d| }dj                  | j                  |fD cg c]  }|s|	 c}      j                         | _        | j
                  j                  |dz   i       j                  di       }t        |j                  d	d            j                         }	d
|	v rd
}
n	d|	v rd}
nd}
d| d|
 d| }dj                  | j                  |fD cg c]  }|s|	 c}      j                         | _        t        | j                        | j                  kD  s[| j                  |       d {    w | j                          y c c}w c c}w 7 "w)Nr[   r|  rV  r   rW  rX  rM  r   step_evaluater   failfailedpending	 | Eval: 	 | Goal: rL  )r   r  r  r  rZ  r   r[  r"  r   rj   r;   rc   r<   r   r   rH  )r  step_ids	last_stepr`  rV  r_  ra  rb  
next_stateraw_eval
evaluations              r@   r  z"Agent._rebuild_memory_from_contextW  s     %'"%'"$,,1134&&( RL	G ..w7;;ORPM%))+6I)#wi'@L-1YYdF`F`bfEg7nEgrkmEg7n-o-u-u-w*++//!R@DD_VXYJ:>>/2>?EEGHH$&
8#%
&
7)9ZL	)MD!%$:L:Ld9S+Z9S2WYB9S+Z![!a!a!cD4%%&););;33'3JJJ+  , 	""$! 8o ,[Js=   B6G18G%
 G%
B)G1-G*
5G*
9<G17G1G/#G1z--brain_stepc                0
   K    j                   t        j                  d j                           dz
  	  j                   _         j
                  j                         }| _        t        j                   j                  d       t        j                  j                   j                  d j                    d      }|j                  |        j                  rt         j                         j                  rt         j                         j                  r"dj                  d  j                  D              nd		 	 d 	 	 	 	 	 d! fd
} |       } j                  j!                           j                  j#                           j                  j%                  |        j                  j'                         } j(                  j+                  |       d {   } j,                  j/                  t1        |j2                              } j,                  j5                  || j                   j(                         d {   \  }}d|vsd|vrt7        d       j9                  || j                          | j:                   j                   <   |d   d    _        |d    _        |d    _         dk\  rt1         j@                  jC                  dd            jE                         }d|v rd}	n	d|v rd}	nd}	 jF                  jI                         D 
cg c]  }
|
jK                         s|
 }}
g }d }|D ]P  }
|
jM                  d d      r(d|
v r#|
jO                  dd      d   jK                         }@|jQ                  |
       R dj                  |      jK                          _#        |d d|	 d| } jR                  jI                         D 
cg c]  }
|
jK                         s|
 }}
|D 
cg c]  }
|
jM                  d d      r|
 }}
|jQ                  |       dj                  |      jK                          _)        tU         jR                         jV                  kD  r jY                         d {    y  j[                          y  j[                          y y 7 7 sc c}
w c c}
w c c}
w 7 =# t\        $ r/}t        j_                  d       ddt1        |      dicY d }~S d }~ww xY ww)"Nu   
📍 Step r   Trh  screenshot_z.pngrM  c              3  2   K   | ]  }t        |        y wrb   )rc   )rd   items     r@   rf   z#Agent.brain_step.<locals>.<genexpr>  s     "K9J3t99Js   Nonec                0   
dk\  rdd d d	j                    ddg}ndddg}| r7|rd	j                  |      nd
}|rd| dnd}|j                  d| |  d       
dk\  rr|j                  ddid       r|j                  ddid       |S )N   r-  Previous step is z).

Recorded info files (filenames only):
z"

Previous Actions Short History:
r   r.  zThis is the first step.

You should provide a JSON with a well-defined goal based on images information. The other fields should be default value.r   r[   zRequested file contents for: rM  zRequested file contents:
	image_urlurlr   r  )r   rj   r!  )read_files_contentread_files_liststate_contentfiles_label
read_label
info_filesprev_step_idprevious_screenshot_dataurlscreenshot_dataurlr  r`  s        r@   build_state_contentz-Agent.brain_step.<locals>.build_state_content  s    a< %+"3L> BJJT VDDHDUDUCVVZ!\	%M %+!\%M &@O$))O"<UWKT_#@R!P  fBJ!(( &&0\2D1E#F*  a<$?!(( +&+-H%I*  &!(( +&+-?%@*  %$rB   rV  analysiszABrain response missing required fields after read-files handling.r?  r   r  r[   r   r  r  r  rW  rY  z| Goal: r  r  rL  z'[Brain] Unexpected error in brain_step.
Brain_textr   )r  reason)NN)r  Optional[str]r  zOptional[list[str]]returnz
list[dict])0r   r   r   r   previous_screenshotr   capture_screenshotr   rr  r   r   rj   rQ   rY   r  brain_message_managerr2  r3  r4  r5  r   r6  r   parse_responserc   r%  maybe_reinvoker   _save_brain_conversationr   r   r   rV  r   r<   r  r\  r;   r]  splitr!  r  r   r   rH  r"  rN  rO  )r  rV   current_screenshot_pathr  r  brain_messagesr(  r$  r  final_statusrb  rc  new_pending	goal_text
final_liner  er  r  r  r  r`  s   `                @@@@@r@   
brain_stepzAgent.brain_step|  s    ,,l4<<.12{t	R'+'@'@D$..AACJ(2D%KK$7&(ggll4??kRVR^R^Q__cDd&e#OO34((%:4;T;T%U"''.CDD\D\.]+OSO`O`"K9J9J"KKflJ487;*%$1*%!4*% *% *%X 01M&&AAC&&CCE&&88G!77DDFN!^^33NCCH&&55c(:J:J6KLF+/+<+<+K+K#**	, &"FN f,
&0H !dee)).&t||)T/5Dt||,#O4[ADN!'
!3D!'!8D q t1155orJKQQS(#,Lx'#+L#,L.2.H.H.S.S.U d.UY[YaYaYc.U d)++/	'B}}u\N"%=>%+(*Q(?(B(H(H(JI &&r* ( .2YY{-C-I-I-K*(#(i~YW`Va!bJ151C1C1N1N1P#_1P2TVT\T\T^B1PL#_1=#m2R]]UZ[gZhhjSkElBL#m ''
3)-<)@)F)F)HD&4--.1C1CC";;,;WWW224..0? !% D&2 !e $`#m X  	RFG IQ"PQQ	Rs   4TF/S ,S-A'S SCS ,S
S
B(S .SSS S)S-A.S SS  T!S 1T2S TS S 
S 	T$$TT	TTTz--actor_stepc           	     	  K   | j                   }d}d }g }|dz
  }	 t        j                  d| j                          | j                  rW| j                         | _        | j                  j                  | j                         d {   }|r|j                         nd}nd}| j                          | j                   dk\  r| j                  r@dd| d| j                   d	| j                   d
| dddt        | j                        idg}ntdd| j                   d| j                   ddddt        | j                        idg}n9dd| j                   d| j                   dddt        | j                        idg}| j                  j                          | j                  j!                          | j                  j#                  ||       | j                  j%                         }	| j'                  |	       d {   \  }}
| j                  | _        | j*                  r| j+                  ||| j                          | j-                  |	|| j                          | j                  j!                          | j                  j/                  |       |r*|j0                  D cg c]  }|j3                  d       c}ng | _        | j6                  j9                  |j0                  | j                  d       d {   }|| _        | j                  rt=        t?        |j0                              D ]c  }dtA        |j0                  |         v st        j                  d       | j                  j                  | j                                d {    e t?        | j4                        dk(  rd| _!        n)dtA        | j4                  d         v rd| _!        nd| _!        | j4                  r4| jB                  s(| jE                          d {    | j                          |r| jK                  |||       | jB                  s| xj                   dz  c_         y y 7 7 ]c c}w 7 h7 7 \# tF        $ r+}| jI                  |       d {  7  }|| _        Y d }~yd }~ww xY w# |r| jK                  |||       | jB                  s| xj                   dz  c_         w w xY ww)Nr[   r   z
Last PID: zNo UI tree found.r  r-  r  z).

Your goal to achieve in this step is: z%

Analysis to the current screen is: z.

Current UI state:
r.  r  r  r  z#Analysis to the current screen is: r   z(. Your goal to achieve in this step is: )	step_infor?  Texclude_unset)action_validopen_appz.Found open_app action, building the tree againr   waitF)&r   r   r&  r  r   r  r   
build_tree&_get_visible_clickable_elements_stringrz  r   r   rY   r   actor_message_managerr3  r2  r4  r5  get_next_actionr   r   _save_actor_conversationadd_model_outputaction
model_dumpr   r   	multi_actr   ranger   rc   r   rd  rN  _handle_step_error_make_history_item)r  r  r`  statemodel_outputresultr  rootr  actor_messagesrawr  ir  s                 r@   
actor_stepzAgent.actor_step  s    ,,%'{d	" LL:dmm_56{{ $ 1 1 3!22==dmmLLIMCCESf ||q ;; %+):<.Hstx  uC  uC  tD DNNRN`N`Maayz  zA(B %0*/1FtG`G`1a)b
%M %+"EdFXFXEY ZIIMHXX\!^ %0*/1FtG`G`1a)b%M  !'%HI[I[H\  ]E  FJ  FT  FT  EU  $V
 !,&+-B4C\C\-]%^	! &&CCE&&AAC&&88T]8^!77DDFN&*&:&:>&J JL#!^^DN..//|T\\R)).,T\\)Z&&AAC&&77Eq}YeYlYl$mYlvV%6%6T%6%JYl$m  DFD!  ??44##%%! 5  F
 !'D{{s<#6#678A!S)<)<Q)?%@@'UW"33>>t?P?P?RSSS 9 4(()Q.&*#3t44Q788&*#&+#$$T-@-@))+++  " ''eVD&&! 'y Mf !K %n T ,  	'221555F &D	' ''eVD&&! 's   SA+Q, QE9Q,  QB$Q, %Q >?Q, =Q%>AQ, AQ, Q(A:Q, Q*Q, "8SQ, Q,  Q, (Q, *Q, ,	R 5R	R
RR# R  R# #9SSc                v  K   t         j                  t        j                        }t	        j
                  ||      }d| j                  dz    d| j                   d}t        |t        t        f      rt         j                  | |        d|v rf| j                  xj                  dz  c_        t         j                  d| j                  j                          | j                  j                          n	d	|v r|d
z  }| xj                  dz  c_        nt        |t               rVt         j#                  | |        t%        j&                  | j(                         d {    | xj                  dz  c_        n.t         j                  | |        | xj                  dz  c_        t+        |d      gS 7 Vw)N)include_traceu   ❌ Result failed r   /z	 times:
 zMax token limit reachedr   z!Reducing agent max input tokens: zCould not parse responsez6

Return a valid JSON object with the required fields.T)errorinclude_in_memory)r   isEnabledForloggingDEBUGr#   format_errorr   r   ry   r   r   r  r  r   r   cut_messagesr   rP  asynciosleepr   r"   )r  r  r  	error_msgprefixs        r@   r  zAgent._handle_step_errord  ss    ++GMM:++EO	%d&?&?!&C%DAdFWFWEXXbceoz:;LLF8I;/0(I5**;;sB;?@Z@Z@k@k?lmn**779+y8WW	%%*%~.NNfXi[12-- 0 0111%%*% LLF8I;/0%%*%9EFF 2s   EF9 F7!AF9c                j    t        |||      }| j                  j                  j                  |       y )N)r  r  r  )r$   r   r!  )r  r  r  r  history_items        r@   r  zAgent._make_history_item  s1     $%

 	##L1rB   z--get_next_actionc                P  K   | j                   j                  |       d{   }t        j                  d|        t	        |j
                        }t        j                  |      }g }|j                  dg       D ]  }t        |t              r|s|j                  |       (t        |j                               d   }t        |t              r||   ni }|dk(  rt        |t              r|j                  dd      }	|j                  dd      }
| j                  j                  |	|
| j                   | j"                  	      }|r)|| j$                  vr| j$                  j                  |       |j                  |        t'        |
      }| j)                  |       ||fS 7 w)zs
        Build a 'structured_llm' approach on top of self.llm. 
        Using the dynamic self.AgentOutput
        NzLLM response: r  r   record_infor-  r[   r0  )rV   r@  )r  )r   r6  r   r&  rc   r%  r   r'  r   ry   r   r!  rm  r[  r   rQ   r   r   r  r&   _log_response)r  input_messagesr(  recordoutput_dictnormalized_actionsr  	outer_keyinner_valueinformation_storedr0  rE  r$  s                r@   r  zAgent.get_next_action  st     *.)?)?)O#O~hZ01X%%&jj(!ooh3Ffd+6"))&1V[[]+A.I/9&$/G&+RKM)jd.K%0__VR%@"'OOK<	!..33&#88	 4 
 *D4E4E"E%%,,Z8%%f-# 4$ &18J%K6"v~7 $Ps   F&F#FF&c                   d| j                   d   v rd}nd| j                   d   v rd}nd}t        j                  | d| j                   d           t        j                  d| j                          t        j                  d	| j                          t        |j                        D ]I  \  }}t        j                  d
|dz    dt        |j                         d|j                  d              K y )NSuccessr  u   ✅Failedu   ❌u   🤷z Eval: u   🧠 Memory: u    🎯 Goal to achieve this step: u   🛠️  Action r   r  z: Tr  )	rV  r   r   r   r   	enumerater  r   model_dump_json)r  r(  emojir  r  s        r@   r  zAgent._log_response  s    **?;;E++O<<EEugWT%7%7%H$IJKmD$5$5#6786t~~6FGH"8??3IAvKK*1q5'3x3G2H6KaKaptKaKuJvwx 4rB   c                   | j                   sy| j                    d| d}t        j                  j                  |      r4t        j                  t        j                  j                  |      d      nd t        |d| j                        5 }| j                  ||       || j                  ||       ddd       t        j                  d|        y# 1 sw Y   "xY w)	z
        Write all the Brain agent conversation (input messages + final AgentOutput)
        into a file: e.g. "brain_conversation_{step}.txt"
        N_brain_r>  Trh  rj  r   zBrain conversation saved to: )r   r   r   r   rr  rs  r   _write_messages_to_file_write_response_to_filer   r   r  r  r(  r@  r0  ry  s         r@   r  zAgent._save_brain_conversation       00889dK	BD''//R[B\BGGOOI.>bf)S4+U+UVZ[((N;#,,Q9 W 	3I;?@ WV   'CCc                   | j                   sy| j                    d| d}t        j                  j                  |      r4t        j                  t        j                  j                  |      d      nd t        |d| j                        5 }| j                  ||       || j                  ||       ddd       t        j                  d|        y# 1 sw Y   "xY w)	z
        Write all the Actor agent conversation (input messages + final AgentOutput)
        into a file: e.g. "actor_conversation_{step}.txt"
        N_actor_r>  Trh  rj  r   zActor conversation saved to: )r   r   r   r   rr  rs  r   r  r  r   r   r  s         r@   r  zAgent._save_actor_conversation  r  r  c                |   |D ]6  }|j                  d|j                  j                   dd d       t        |j                  t
              r|j                  D ]  }t        |t              s|j                  d      dk(  rI|j                  d      xs |j                  dd      }|j                  d|j                          d       q|j                  d      d	k(  s|d	   d
   }|j                  d|dd  d        n'|j                  t        |j                         d       |j                  d       9 y)z
        For each message, write it out in a human-readable format.
        Or adapt your existing logic from _write_messages_to_file.
        rM  z(----------------------------------------r   r-  r%  r[   z[Text Content]
r   r  r  z[Image URL]
Nd   z...

>
============================================================
)
rv  rg   rh   ry   r%  rm  r   r   r;   rc   )r  ry  messagesmessager  txtr  s          r@   r  zAgent._write_messages_to_file  s   
  GGGb**334BvhbAB'//40#OOD!$-88F+v5"&((9"5"M&"9MCGG&6syy{m4$HI!XXf-<(,[(9%(@IGGmIdsO3DG$LM , 3w/056GG()  rB   c                    |j                  d       |j                  t        |      dz          |j                  d       y)z>
        If the AgentOutput is JSON-like, you can do:
        z	RESPONSE
rM  r  N)rv  rc   )r  ry  r(  s      r@   r  zAgent._write_response_to_file  s4     	
 	
H$%	$%rB   c                H    t         j                  d| j                          y )Nu   🚀 Starting task: )r   r   r:   r  s    r@   _log_agent_runzAgent._log_agent_run  s    *499+67rB   c                  K   	 | j                          | j                  r$| j                  s| j                          d {    t	        |      D ]   }| j                  r| j                          d {    d| _        | j                         r | j                  S | j                          d {   s | j                  S | j                          d {    | j                          d {    | j                  j                         rIt        j                  d       | j                  r| j                  | j                          | j                  S t        j                   d       d {    # t        j                  d       | j                  S 7 U7 %7 7 7 7 4# t"        $ r t        j%                  d        w xY ww)NFu   ✅ Task completed successfullyr  u,   ❌ Failed to complete task in maximum stepszError running agent)r
  r   r   editr  r  _too_many_failures_handle_control_flagsr  r  r   is_doner   r   r   r  r  rN  rO  )r  	max_stepsr@  s      r@   runz	Agent.run  s{    	!iik!!i(;;**,,,"'DK**,  << "77999 << oo'''oo'''<<'')KK AB2233DLLA
 <<	 mmA&&&# )& JK<</ " - : (' '
  	23	s   G;F" F 2F" 2F3'F" GF" .F/F" ?G F" FF" +F,A&F" GF" +F ,'F" GF" F" F" F" F"  F" " GGc                   K   | j                   j                          d {   }| j                  |j                  |j                         y 7 +wrb   )r   	edit_task_set_new_taskraw_textpayload)r  r  s     r@   r  z
Agent.edit4  s8     ||--//6??FNN; 0s   AA,AzThe overall user's task is: zThe step by step plan is: c                   |}t        |t              r| j                  |      }|j                  | j                        r|}n*| j                   | j
                   d| j                   d| }| j                  r7| j                  r*g }t        |t              r|j                  dg       xs g }t        |t              r;|D cg c]/  }t        |t              s|j                         s%t        |      1 }}ng }|| _        | j                  r0t        j                  ddj!                  | j                               nt        j                  d       t#        | j                  | j                  | j$                  xs d      }t'        |      | _        | j(                  r| d| j(                   }|| _        | j-                          yc c}w )	z
        Build the final task string:
            "The overall plan is: <original task>

<generated plan>"
        and update every MessageManager in one go.
        rM  r   zPlanner selected skills: %sr   zPlanner selected no skills.N)	max_charsz$

Selected skills (planner-chosen):
)ry   r   _format_plan_payloadr]  PREFIXr   SUFFIXr   r   r   rm  rc   r;   r   r   r   rj   r,   r   r.   r   r:   r   )r  generated_planplan_payload	plan_text
final_taskselectedsskill_contentss           r@   r  zAgent._set_new_task;  s    #	lD)11,?I$$T[[1'J KK=););(<Bt{{m2i[YJ??t44H,-'++,=rBHb(D),4YHq
1c8JqwwyCFHY#+D ##9499TEYEY;Z[9:0%%$$//74N
 "6n!ED!!!l"I))*, 
 	 - Zs   GG)Gc                   g }|j                  d      }t        |t              r=|j                  d      }|j                  d      }|r|r|j                  d| d|        |j                  d      }t        |t              r2|j                         r"|j                  d|j                                 |j                  d      }t        |t              r_|D cg c]/  }t        |t              s|j                         s%t	        |      1 }	}|	r#|j                  d	d
j                  |	              |j                  d      }
t        |
t              rA|
j                         r1|j                  d       |j                  |
j                                n|j                  d      }t        |t              r|r|j                  d       |D ]o  }t        |t              s|j                  d      xs d}|j                  d      xs d}|sA|r|j                  d| d| d       \|j                  d|        q |rdj                  |      S t        j                  |d      S c c}w )Niteration_infocurrent_iterationtotal_iterationszIteration: r  search_summaryzSearch summary: r   zSelected skills: r   natural_language_planzPlan:step_by_step_plandescriptionr[   important_search_infoz- z
 (search: )rM  F)rp  )
r   ry   r   r!  rc   r;   rm  rj   r   rw  )r  r  r  	iterationcurrenttotalr'  r   r!  selected_cleannatural_planstepsr@  descr   s                  r@   r  zAgent._format_plan_payloadf  s   KK 01	i&mm$78GMM"45E5{7)1UG<= %56nc*~/C/C/ELL+N,@,@,B+CDE;;01h%.6[h*Q:LQRQXQXQZc!fhN[0>1J0KLM{{#:;lC(\-?-?-ALL!LL++-.KK 34E%&5W%!D%dD1 88M28bD88$;<BD r$z$q%ABr$[1 " $)tyyUdjju.UU1 \s   I(/I( I(c                ~    | j                   | j                  k\  r$t        j                  d| j                   d       yy)Nu   ❌ Stopping due to z consecutive failuresTF)r   r   r   r  r  s    r@   r  zAgent._too_many_failures  s:    $$(9(99LL/0A0A/BBWXYrB   c                   K   | j                   rt        j                  d       y| j                  r7t	        j
                  d       d {    | j                   ry| j                  r7y7 w)NzAgent stoppedFg?T)r   r   r   r   r  r  r  s    r@   r  zAgent._handle_control_flags  sR     ==KK(ll--$$$}} ll
 	 %s   AA+	A)
A+(A+c                B    |rt         j                  d|       d| _        y )NzStopping agent: %sT)r   rP  r   )r  r  s     r@   stopz
Agent.stop  s    NN/8rB   c                B    |sd}| j                   j                  |       y )NzAgentHistory.json)r   save_to_file)r  	file_paths     r@   save_historyzAgent.save_history  s    +I!!),rB   c                   t        | j                  | j                  | j                  j                  j                         t        | j                  | j                  | j                  | j                  d	      | _        t        | j                  | j                  | j                  j                  j                         t        | j                  | j                  | j                  | j                  d	      | _        t        | j                  | j                  | j                  j                  j                         t         | j                  | j                  | j                  | j                  d	      | _        y )NT)	rk   r:   action_descriptionssystem_prompt_classr   r   r   r   	give_taskF)r   r   r:   r   r  get_prompt_descriptionr   r   r   r   r   r  r   r   r  r   r    r1  r  s    r@   r   zAgent.initiate_messages  s   %3 $ 8 8 O O Q 1!22#66!22!%!:!:
&
" &4 $ 8 8 O O Q 1!22#66!22!%!:!:
&
" '5 $ 8 8 O O Q ,!22#66!22!%!:!:
'
#rB   ):r:   rc   r   r   r   r   r   r   r   r0   r   rx   r   rx   r   r  r   r  r   Optional[BaseChatModel]r   r  r   r  r   r  r   r  r   r  r   r  r   r  r   r  r   r  r   Optional[int]r   r  r   r  r   z	list[str]r   r  r   r  r   z2Callable[['str', 'AgentOutput', int], None] | Noner   z+Callable[['AgentHistoryList'], None] | Noner  r  r   r  )r  r  )r  r  r  r  )r  rB  )r(  r   r  r   )r)  rc   r7  rc   r  ztuple[str, str]rb   )
r)  rc   r0  rc   rA  rc   rB  rB  r  r  )rB  rB  r  r  )r  r   )r  zOptional[AgentStepInfo]r  r  )r  rN  r  list[ActionResult])r  zAgentOutput | Noner  rc   r  rC  r  r  )r  list[BaseMessage]r  r&   )r(  r&   r  r  )r  rD  r(  r   r@  r  r  r  )ry  r   r  rD  r  r  )ry  r   r(  r   r  r  )r  )r  r  r  r%   )r  rc   r  zOptional[dict]r  r  )r  r   r  rc   )r  rx   )r  r  r  r  )r:  zOptional[str | Path]r  r  )+rh   
__module____qualname__r0   r  r  r  r   r  r"  r+  r:  rF  rI  rH  rQ  rd  rz  r  r  r2   r  r  r  r  r  r  r  r  r  r  r
  r  r  r  r  r  r  r  r  r7  r;  r   ru   rB   r@   r   r      sj    ", $( $/38<AH6:?F6:?F'+ /3 %)
 !$$&Y]NR-3"&Yn"n" !n" !	n"
 "n" n" n" n" "n" n" -n" )6n" 2?n" '4n"  0=!n"" '4#n"$ 0=%n"& %'n"( )n"* +n",  --n". /n"0 1n"4 &5n"L Mn"N "On"R %WSn"T !LUn"V +Wn"X  Yn"b(	R
7
3", (,  	
 % 
*.%8%.%.y6.?`#%J .)xR *xRt .)j" *j"XG62(2 2 #	2
 
2 -.  / FyA)A A 	A
 
A2A)A A 	A
 
A2**
&8B< ,F)F)!V'VR

-
!
rB   r   )r:   rc   r?   r  r  rc   )r:   rc   rF   r   r  rc   )rk   rA  r  rc   )rk   rA  r  rx   )rk   rA  r  rA  )`
__future__r   r  rR   rO   r   r  r   pathlibr   Quartztypingr   r   r   r   r	   r
   r   r=   r   dotenvr   *langchain_core.language_models.chat_modelsr   collectionsr   langchain_openair   r   langchain_anthropicr   langchain_google_genair   langchain_ollamar   langchain_core.messagesr   lmnrr   openair   PILr   r   r   pydanticr   r   !src.agent.message_manager.servicer   src.agent.promptsr   r   r    r!   src.agent.viewsr"   r#   r$   r%   r&   r'   r(   src.utils.record_storer)   src.utils.brain_searchr*   src.utils.skillsr+   r,   r-   r.   src.agent.planner_servicer/   src.controller.servicer0   src.mac.treer1   	src.utilsr2   src.agent.output_schemasr3   src.agent.structured_llm	getLoggerrh   r   r5   TASK_ID_MAX_LENrA   rI   rY   rm   r}   r   r   ru   rB   r@   <module>re     s    "   	   	   E E E 	   D  # D 8 - 9 '  ! + + / <    / 2  . - ) * 2 & 			8	$Cy!,;  
5
A9v_
 _
rB   