
    Sg@]                        d dl Z d dlZd dlZd dlZd dlmZmZmZmZm	Z	m
Z
 d dlmZmZmZ d dlZd dlZd dlZd dlZd dlmZmZ d dlmZ d dlmZmZmZmZmZ d dlmZ d dl m!Z! d d	l"m#Z# d d
l$m%Z% d dl&m'Z'm(Z(m)Z) d dl*m+Z+ d dl,m-Z- d dl.m/Z/m0Z0m1Z1 d dlm2Z2 d dl3m4Z4 d dl5m6Z6 ddl7m8Z8 ddl9m:Z:m;Z;  ej<        e=          Z> G d de          Z?dS )    N)OptionalTypeListDictAnyCallable)Image	ImageDraw	ImageFont)SystemPromptAgentMessagePrompt)Agent)ActionResultActionModelAgentHistoryListAgentOutputAgentHistory)Browser)BrowserContext)BrowserStateHistory)
Controller)AgentEndTelemetryEventAgentRunTelemetryEventAgentStepTelemetryEvent)time_execution_async)BaseChatModel)BaseMessageHumanMessage	AIMessage)PlannerPrompt)repair_json)
AgentState   )CustomMessageManager)CustomAgentOutputCustomAgentStepInfoc            ?       (    e Zd Zddd e            ddddddeeddddddg d	d
dddddddddfdededededz  de	dz  dede
de
dee         dee         dededee         dee         dede
dee         de
ez  deeeef                  d eee                  d!ee         d"ed#ed$e
d%eeeeeeef         f                           d&ed'd(egdf         dz  d)ed*gdf         dz  d+ee         d,ee         d-ee         d.ef> fd/ZdQd1Zd2ed0dfd3Z	 dRd4ed5efd6Z ed7          d8ee         d0efd9            Zd0ee         fd:Z ed;          dRd5ee         d0dfd<            ZdSd>ed0efd?Z 	 	 	 	 	 	 	 	 	 	 dTdFedGedHe
dIe
dJe
dKedLedMedNedOe!d0dfdPZ" xZ#S )UCustomAgent NTFzutf-8   
   i  )
titletypenameroletabindexz
aria-labelplaceholdervaluealtzaria-expandedi  autor#   taskllm	add_infosbrowserbrowser_context
controller
use_visionuse_vision_for_plannersave_conversation_pathsave_conversation_path_encodingmax_failuresretry_delaysystem_prompt_classagent_prompt_classmax_input_tokensvalidate_outputmessage_contextgenerate_gifsensitive_dataavailable_file_pathsinclude_attributesmax_error_lengthmax_actions_per_steptool_call_in_contentinitial_actionsregister_new_step_callbackBrowserStater   register_done_callbackr   tool_calling_methodpage_extraction_llmplanner_llmplanner_intervalc                     i } t           j                                        D ]H\  }!}"|!                    d          r.|!                    ddd                                          }#|"| |#<   I|i }i | |}t                                          ||||||||	|
|||||||||||||||||||           | j        dv s	d| j        v rd| _	        d| _
        nd	| _	        d | _        d| _        || _        || _        t          | j        | j        | j        j                                        | j        || j
        | j        | j        | j        | j        | j        
          | _        d S )N
SENSITIVE_r)   r#   )r5   r6   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rP   rQ   rS   rT   )deepseek-reasonerzdeepseek-r1Ti   F)r6   r5   action_descriptionsrA   rB   rC   rI   rJ   rK   rE   rG   )osenvironitems
startswithreplacelowersuper__init__
model_nameuse_deepseek_r1rC   _last_actionsextracted_contentr7   rB   r$   r6   r5   r:   registryget_prompt_descriptionrA   rI   rJ   rK   rE   rG   message_manager)%selfr5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rP   rQ   rR   rS   rT   env_sensitive_datakeyr2   env_key	__class__s%                                       F/Users/kosar/src/browser-use-projects/web-ui/src/agent/custom_agent.pyr`   zCustomAgent.__init__0   s   `  ***,, 	4 	4JC~~l++ 4++lB::@@BB.3"7+ !NA.A.A 	+!!#9#9,K%# 3-++%)!51-!5!5+'A#9 3#-9 	 	
 	
 	
< ?333}7W7W#'D $)D!!#(D  "!#""43 $ 8 O O Q Q $ 81!2#6!2!%!: 0. 
  
  
    returnc                     | j         j                                        | _        t	          j        | j                  | _        dS )z6Setup dynamic action models from controller's registryN)r:   re   create_action_modelr   r%   type_with_custom_actionsr   rh   s    rm   _setup_action_modelsz CustomAgent._setup_action_models   s9      ?3GGII,EdFVWWrn   responsec                    d|j         j        v rd}nd|j         j        v rd}nd}t                              | d|j         j                    t                              d|j         j                    t                              d|j         j                    t                              d	|j         j                    t                              d
|j         j                    t                              d|j         j                    t          |j
                  D ]Q\  }}t                              d|dz    dt          |j
                   d|                    d                      RdS )zLog the model's responseSuccessu   ✅Failedu   ❌u   🤷z Eval: u   🧠 New Memory: u   ⏳ Task Progress: 
u   📋 Future Plans: 
u   🤔 Thought: u   🎯 Summary: u   🛠️  Action r#   /z: Texclude_unsetN)current_stateprev_action_evaluationloggerinfoimportant_contentstask_progressfuture_plansthoughtsummary	enumerateactionlenmodel_dump_json)rh   ru   emojiir   s        rm   _log_responsezCustomAgent._log_response   s   .EEEEE/FFFEEEuTTX%;%RTTUUUS(>(QSSTTTRH,B,PRRSSSQH,B,OQQRRREX%;%CEEFFFEX%;%CEEFFF"8?33 	 	IAvKKo1q5oo3x+?+?oo6CYCYhlCYCmCmoo   	 	rn   model_output	step_infoc                 2   |dS |xj         dz  c_         |j        j        }|r d|vr||j        vr|xj        |dz   z  c_        |j        j        }|rd|vr||_        |j        j        }|rd|vr||_        t                              d|j                    dS )z"
        update step info
        Nr#   None
u   🧠 All Memory: 
)step_numberr|   r   memoryr   r   r~   r   )rh   r   r   r   r   r   s         rm   update_step_infozCustomAgent.update_step_info   s     F")7J"	:"444&i.>>> 2T 99$2@ 	4V=88&3I##1> 	2F,66%1I"<)*:<<=====rn   z--get_next_actioninput_messagesc                 &  K   | j                             |          }| j                            |           t	          |d          rSt
                              d           t
                              |j                   t
                              d           t          |j	        t                    r|j	        d         }n|j	        }|                    dd                              dd          }t          |          }t          j        |          } | j        di |}|.t
                              |j	                   t#          d	          |j        d| j                 |_        |                     |           | xj        d
z  c_        |S )z/Get next action from LLM based on current statereasoning_contentu   🤯 Start Deep Thinking: u   🤯 End Deep Thinkingr   ```jsonr)   ```NzCould not parse response.r#    )r6   invokerg   _add_message_with_tokenshasattrr~   r   r   
isinstancecontentlistr]   r!   jsonloadsr   debug
ValueErrorr   rK   r   n_steps)rh   r   
ai_message
ai_contentparsed_jsonparseds         rm   get_next_actionzCustomAgent.get_next_action   sp      X__^44
55jAAA:233 	2KK4555KK
4555KK0111j($// 	,#+A.JJ#+J''	266>>ubII
 ,,
j,,.d.====>LL+,,,8999 &A(A&AB6"""rn   c                 Z  K   | j         sdS t          | j                                                  g| j                                        dd         }| j        su| j        rn|d         }d}t          |j	        t                    r0|j	        D ]'}|d         dk    r||d         z  }|d         dk    r'(n|j	        }t          |          |d<   | j                             |           d{V }|j	        }|d         }t          |j	        t                    r+|j	        D ]"}|d         dk    r|dxx         d	| d
z  cc<   #n|xj	        d	| dz  c_	        	 t          j        |                    dd                              dd                    }t                               dt          j        |d                      t'          |d          rUt                               d           t                               |j                   t                               d           dS dS # t          j        $ r! t                               d|            Y dS t,          $ rE}t                               d|            t                               d|            Y d}~dS d}~ww xY w)z7Run the planner to analyze state and suggest next stepsNr#   r)   r-   text	image_url)r   z 
Planning Agent outputs plans:
 r   z
 r   r   u   📋 Plans:
   )indentr   u#   🤯 Start Planning Deep Thinking: u   🤯 End Planning Deep Thinkingz!Error parsing planning analysis: u   📋 Plans: )rS   r    rX   get_system_messagerg   get_messagesr<   r;   r   r   r   r   ainvoker   r   r]   r~   r   dumpsr   r   JSONDecodeError	Exceptionr   )	rh   planner_messageslast_state_messagenew_msgmsgru   plan	plan_jsones	            rm   _run_plannerzCustomAgent._run_planner   s&       	4 $233FFHH
!..004

 * 	At 	A!1"!5G,4d;; 5-5 ! !C6{f,,3v;.V33  4! -4#/#@#@#@R  )112BCCCCCCCC-b1(0$77 	Y)1 Q Qv;&((KKK#P#P#P#PPKKKQ &&*Xt*X*X*XX&&	/
4<<	2#>#>#F#Fub#Q#QRRIKKI
9Q(G(G(GIIJJJx!455 ?ABBBH6777=>>>>>? ?
 # 	0 	0 	0KK...////// 	/ 	/ 	/LL@Q@@AAAKK-t--.........	/s   CH- -,J*	J*%:J%%J*z--stepc                 Z	   K   t                               d j                    d}d}g }g }	  j                                         d{V }                                   j                            | j         j	        | j
                    j        r- j         j        z  dk    r                                  d{V   j                                        }                                  	                      |           d{V } j        r                     || j                                        ||                                ||            j        dk    r j                            d                                             n.# t*          $ r!} j                            d           |d}~ww xY w|j        } j                            | j         j         j         fd j                   d{V }t9          |          t9          |          k    rt;          t9          |          t9          |                    D ]q}|                    t?          dd||                              d	           d
|t9          |          dz
                                d	           d                     r|D ]'}	|	j!        rd|	j!        v r xj!        |	j!        z  c_!        (| _	        | _        t9          |          dk    rZ|d         j"        rM j!        s|j#         _!         j!        |d         _!        t                               d|d         j!                    d _$        n9# t*          $ r,} %                    |           d{V }| _	        Y d}~nd}~ww xY w|rd |j        D             ng } j&        '                    tQ           j)         j        | j$        |rd |D             ndg                     |sdS |r *                    |||           dS dS # |rd |j        D             ng } j&        '                    tQ           j)         j        | j$        |rd |D             ndg                     |s Y dS |r *                    |||           w w xY w)zExecute one step of the tasku   
📍 Step Nr   rW   r   c                  ,                                      S N_check_if_stopped_or_pausedrs   s   rm   <lambda>z"CustomAgent.step.<locals>.<lambda>`  s    d.N.N.P.P rn   )rR   rG   check_break_if_pausedrH   Trz   zo is Failed to execute.                                                     Something new appeared after action r#   F)rd   include_in_memoryerroris_donezExtracted pageu   📄 Result: c                 :    g | ]}|                     d           S )Trz   )
model_dump).0as     rm   
<listcomp>z$CustomAgent.step.<locals>.<listcomp>~  s&    UUUAq||$|77UUUrn   c                 *    g | ]}|j         	|j         S r   )r   )r   rs     rm   r   z$CustomAgent.step.<locals>.<listcomp>  s!    CCCA17CCCCrn   z	No result)agent_idstepactionsconsecutive_failures
step_error)+r~   r   r   r9   	get_stater   rg   add_state_messagerc   _last_resultr;   rS   planning_intervalr   r   r   rN   r   _save_conversationra   _remove_state_message_by_indexr   r   r:   	multi_actrR   rG   rH   r   rangeappendr   r   rd   r   r   r   _handle_step_error	telemetrycapturer   r   _make_history_item)
rh   r   stater   resultr   r   r   riret_s
   `         rm   r   zCustomAgent.step6  s      	14<11222%'%'M	E.88::::::::E,,... 225$:LdN_aj37?D D D  *DL43I$IQ$N$N'')))))))))!1>>@@N,,...%)%9%9.%I%IIIIIII2 W33E<VVV%%lI>>>''EEE?&999(GGKKK002222   $CCBGGG
 *6)<G/3/H/H$$($<#2&P&P&P&P%)%> 0I 0 0 * * * * * *F 6{{c'll**FS\\:: ? ?BMM,AE8?8S8Sbf8S8g8g 6X 6XY`adekalalopapYq  ZB  ZB  QU  ZB  ZV  ZV6X 6X7<	#> #> #> ? ? ? ?
  E E) E.>$BX.X.X**d.DD** &D!(D6{{Q6":#5- >-6-=D*/3/Er
,JF2J,HJJKKK()D%% 	' 	' 	'22155555555F &D	'
 ZfmUUATUUUUkmGN""'!]#)-)BGM`CCCCCCT_S`       E''eVDDDDDE E ZfmUUATUUUUkmGN""'!]#)-)BGM`CCCCCCT_S`       E''eVDDDDEs]   CM0 1B#F M0 
G F;;G  F/M0 /P) 0
N&:"N!P) !N&&P) )A#R*R*d   	max_stepsc                 J   K   	                                     j        rD j                             j         j        d j         fd j                   d{V }| _        t           j	         j
        d|ddd          }t          |          D ]}                                 r n                     |           d{V   j                                        rG j        r$||dz
  k     r                                  d{V swt$                              d            nqt$                              d	            j        s(|j         j        j        d
         j        d
         _        n' j         j        j        d
         j        d
         _         j         j                            t3           j         j                                         j         j        |k     j                                                              j        s j                                         d{V   j        s& j         r j                                          d{V   j!        r:d}tE           j!        tF                    r j!        } $                    |           S S #  j                            t3           j         j                                         j         j        |k     j                                                              j        s j                                         d{V   j        s& j         r j                                          d{V   j!        r:d}tE           j!        tF                    r j!        } $                    |           w w xY w)z-Execute the task with maximum number of stepsFc                  ,                                      S r   r   rs   s   rm   r   z!CustomAgent.run.<locals>.<lambda>  s    $2R2R2T2T rn   )check_for_new_elementsrR   r   rH   Nr#   r)   )r5   r7   r   r   r   r   r   u   ✅ Task completed successfullyu,   ❌ Failed to complete task in maximum stepsr   )r   successstepsmax_steps_reachederrorsagent_history.gif)output_path)%_log_agent_runrM   r:   r   r9   rR   rH   r   r&   r5   r7   r   _too_many_failuresr   historyr   rD   _validate_outputr~   r   rd   r   r   r   r   r   r   r   r   injected_browser_contextcloseinjected_browserr8   rF   r   strcreate_history_gif)rh   r   r   r   r   r   s   `     rm   runzCustomAgent.run  s<     H	A!!! # 	+#88((+0(,(@*T*T*T*T)-)B  9           %+!+Y.#   I i(( c c**,, E ii	*********<''))  0%59IM5I5I%)%:%:%<%<<<<<<< %$KK ABBBE JKKK- cLUL\DL(,3B7IILPLbDL(,3B7I< N""&!] L0022,&*li&?<..00     0 3*00222222222( +T\ +l((*********  A#6d/55 4"&"3K''K'@@@@A! N""&!] L0022,&*li&?<..00     0 3*00222222222( +T\ +l((*********  A#6d/55 4"&"3K''K'@@@@As   FJ DN"r     (   8   ,         ?r   duration
show_goals	show_task	show_logo	font_sizetitle_font_sizegoal_font_sizemarginline_spacingc           
         | j         j         st                              d           dS g }| j         j         r| j         j         d         j        j        st                              d           dS 	 g d}d}|D ]}	 t          j                    dk    r6t          j        	                    t          j
        dd	          |d
z             }t          j        ||          }t          j        ||          }t          j        ||          }d} n# t          $ r Y w xY w|st          d          n8# t          $ r+ t          j                    }t          j                    }|}Y nw xY wd}|r	 t          j        d          }d}|j        |j        z  }t'          ||z            }|                    ||ft          j        j                  }n4# t.          $ r'}t                              d|            Y d}~nd}~ww xY w|rU| j        rN|                     | j        | j         j         d         j        j        ||||
          }|                    |           t7          | j         j         d          D ]\  }}|j        j        st9          j        |j        j                  }t          j        t=          j        |                    }|r2|j         r+| !                    |||j         j"        j#        |||	|          }|                    |           |rH|d         $                    |d|dd         |dd           t          %                    d|            dS t                              d           dS )zGCreate a GIF from the agent's history with overlaid task and goal text.zNo history to create GIF fromNr   z1No history or first screenshot to create GIF from)	HelveticaArial
DejaVuSansVerdanaFWindowsWIN_FONT_DIRzC:\Windows\Fontsz.ttfTzNo preferred fonts foundz./static/browser-use.png   zCould not load logo: r#   )imager   	goal_textregular_font
title_fontr  logo)save_allappend_imagesr   loopoptimizezCreated GIF at z(No images found in history to create GIF)&r   r~   warningr   
screenshotplatformsystemrY   pathjoingetenvr   truetypeOSErrorload_defaultr	   openwidthheightintresize
ResamplingLANCZOSr   r5   _create_task_framer   r   base64	b64decodeioBytesIOr   _add_overlay_to_imager|   r   saver   )rh   r   r   r   r  r  r  r  r  r  r  imagesfont_optionsfont_loaded	font_namer  r  	goal_fontr  logo_heightaspect_ratio
logo_widthr   
task_framer   itemimg_datar  s                               rm   r   zCustomAgent.create_history_gif  s    |# 	NN:;;;F|# 	4<+?+B+H+S 	NNNOOOF	%JJJLK)  	
((I55$&GLL>K_1`1`bkntbt$u$u	#,#5i#K#KL!*!3I!O!OJ ) 29n M MI"&KE   H  :8999:  	% 	% 	%$133L"/11J$III		%  		<<z"<==!#zDK7 |!;<<
{{J#<e>N>VWW < < <:q::;;;;;;;;<  		& 		&00	$Q'-8 J MM*%%% !!5q99 	! 	!GAt:(  '
(=>>HJrz(3344E 	d/ 	22 !"/=E!-)! 3   MM%     	G1INN$QRRj!     KK7+7788888NNEFFFFFsP   0
D- ;BD	D- 
DD- DD- -2E"!E"*AG	 	
G:G55G:)ro   Nr   )r   )
r   r   TTFr   r   r   r   r   )$__name__
__module____qualname__r   r   r   r   r   r   r   boolr   r&  r   r   r   r   r   r   r`   rt   r%   r   r&   r   r   r   r   r   r   r   r   r   floatr   __classcell__)rl   s   @rm   r(   r(   /   s       
  &*59%/Z\\#+048=D !!6B;M$*$)-1'+7;8<- - - %((*)-IMfjRV17;?37$%Yv
 v
v
 v
 	v

 t^v
 ,d2v
 #v
 v
 %)v
 %-SMv
 .6c]v
 v
 v
 "&l!3v
 !%%7 8v
  "!v
" "#v
$ &c]%v
& *'v
( %T#s(^4)v
* #+49"5+v
, !%S	-v
D "Ev
F #&Gv
H #'Iv
J &d4T#s(^0C+D&EFKv
N )1.-QT1UW[1[(\_c(cOv
P %-.@-A4-G$H4$OQv
R "*#Sv
T "*-!8Uv
V "-0Wv
X "Yv
 v
 v
 v
 v
 v
pX X X X&7 D    * UY> > 1>>Q> > > >8 -..D4E +    /.@4/HSM 4/ 4/ 4/ 4/l (##UE UEH-@$A UET UE UE UE $#UEnJA JA3 JA1A JA JA JA JA\  3 #"##%"$"%oG oGoG oG 	oG
 oG oG oG !oG  oG oG  oG 
oG oG oG oG oG oG oG oGrn   r(   )@r   loggingpdb	tracebacktypingr   r   r   r   r   r   PILr	   r
   r   rY   r+  r-  r  browser_use.agent.promptsr   r   browser_use.agent.servicer   browser_use.agent.viewsr   r   r   r   r   browser_use.browser.browserr   browser_use.browser.contextr   browser_use.browser.viewsr   browser_use.controller.servicer   browser_use.telemetry.viewsr   r   r   browser_use.utilsr   *langchain_core.language_models.chat_modelsr   langchain_core.messagesr   r   r   r    json_repairr!   src.utils.agent_stater"   custom_message_managerr$   custom_viewsr%   r&   	getLoggerr<  r~   r(   r   rn   rm   <module>rW     s     



     < < < < < < < < < < < < < < < < + + + + + + + + + + 				  				  F F F F F F F F + + + + + +              0 / / / / / 6 6 6 6 6 6 9 9 9 9 9 9 5 5 5 5 5 5         
 3 2 2 2 2 2 D D D D D D         
 4 3 3 3 3 3 # # # # # # , , , , , , 8 8 8 8 8 8 @ @ @ @ @ @ @ @		8	$	$ZG ZG ZG ZG ZG% ZG ZG ZG ZG ZGrn   