
    nFgs                        d dl Z d dlZd dlZd dlZd dlmZmZmZmZm	Z	m
Z
 d dlmZmZmZ d dlZd dlZd dlZd dlZd dlmZmZ d dlmZ d dlmZmZmZmZmZ d dlmZ d dl m!Z! d d	l"m#Z# d d
l$m%Z% d dl&m'Z'm(Z(m)Z) d dl*m+Z+ d dl,m-Z- d dl.m/Z/m0Z0m1Z1 d dlm2Z2 d dl3m4Z4 d dl5m6Z6 ddl7m8Z8 ddl9m:Z: ddl;m<Z<m=Z= d dl>Z> ej~                  e@      ZA G d de      ZBy)    N)OptionalTypeListDictAnyCallable)Image	ImageDraw	ImageFont)SystemPromptAgentMessagePrompt)Agent)ActionResultActionModelAgentHistoryListAgentOutputAgentHistory)Browser)BrowserContext)BrowserStateHistory)
Controller)AgentEndTelemetryEventAgentRunTelemetryEventAgentStepTelemetryEvent)time_execution_async)BaseChatModel)BaseMessageHumanMessage	AIMessage)PlannerPrompt)repair_json)
AgentState   )enhanced_repair_json)CustomMessageManager)CustomAgentOutputCustomAgentStepInfoc            ?           e Zd Zddd e       ddddddeeddddddg d	d
dddddddddfdededededz  de	dz  dede
de
dee   dee   dededee   dee   dede
dee   de
ez  deeeef      d eee      d!ee   d"ed#ed$e
d%eeeeeeef   f         d&ed'd(egdf   dz  d)ed*gdf   dz  d+ee   d,ee   d-ee   d.ef> fd/ZdJd1Zd2ed0dfd3Z	 dKd4ed5efd6Z ed7      d8ee   d0efd9       Zd0ee   fd:Z ed;      dKd5ee   d0dfd<       ZdLd=ed0efd>Z 	 	 	 	 	 	 	 	 	 	 dMd?ed@edAe
dBe
dCe
dDedEedFedGedHe!d0dfdIZ" xZ#S )NCustomAgent NTFzutf-8   
   i  )
titletypenameroletabindexz
aria-labelplaceholdervaluealtzaria-expandedi  autor#   taskllm	add_infosbrowserbrowser_context
controller
use_visionuse_vision_for_plannersave_conversation_pathsave_conversation_path_encodingmax_failuresretry_delaysystem_prompt_classagent_prompt_classmax_input_tokensvalidate_outputmessage_contextgenerate_gifsensitive_dataavailable_file_pathsinclude_attributesmax_error_lengthmax_actions_per_steptool_call_in_contentinitial_actionsregister_new_step_callbackBrowserStater   register_done_callbackr   tool_calling_methodpage_extraction_llmplanner_llmplanner_intervalc                     i } t         j                  j                         D ]=  \  }!}"|!j                  d      s|!j	                  ddd      j                         }#|"| |#<   ? |i }i | |}t        $|   ||||||||	|
|||||||||||||||||||       t        | d      s9t        | j                  d      r| j                  j                  | _
        nd| _
        | j                  dv sd| j                  v rd| _        d	| _        nd
| _        d | _        d| _        || _        || _        t#        | j                  | j$                  | j&                  j(                  j+                         | j,                  || j                  | j.                  | j0                  | j2                  | j4                  | j6                        | _        y )N
SENSITIVE_r*   r#   )r6   r7   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rQ   rR   rT   rU   
model_name)deepseek-reasonerzdeepseek-r1Ti   F)r7   r6   action_descriptionsrB   rC   rD   rJ   rK   rL   rF   rH   )osenvironitems
startswithreplacelowersuper__init__hasattrr7   rX   use_deepseek_r1rD   _last_actionsextracted_contentr8   rC   r%   r6   r;   registryget_prompt_descriptionrB   rJ   rK   rL   rF   rH   message_manager)%selfr6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rQ   rR   rS   rT   rU   env_sensitive_datakeyr3   env_key	__class__s%                                       F/Users/kosar/src/browser-use-projects/web-ui/src/agent/custom_agent.pyrb   zCustomAgent.__init__3   s   `  ****, 	4JC~~l+++lB:@@B.3"7+	4 !NA.A.A 	+!!#9#9,K%# 3-++%)!51-!5!5+'A#9 3#-9 	 	
> t\*txx."&(("5"5 #%??33}7W#'D $)D!#(D  "!#""43 $ 8 8 O O Q $ 8 81!22#66!22!%!:!: 00.. 
    returnc                     | j                   j                  j                         | _        t	        j
                  | j                        | _        y)z6Setup dynamic action models from controller's registryN)r;   rg   create_action_modelr   r&   type_with_custom_actionsr   rj   s    ro   _setup_action_modelsz CustomAgent._setup_action_models   s8      ??33GGI,EEdFVFVWrp   responsec                 J   d|j                   j                  v rd}nd|j                   j                  v rd}nd}t        j                  | d|j                   j                          t        j                  d|j                   j                          t        j                  d|j                   j
                          t        j                  d	|j                   j                          t        j                  d
|j                   j                          t        j                  d|j                   j                          t        |j                        D ]I  \  }}t        j                  d|dz    dt        |j                         d|j                  d              K y)zLog the model's responseSuccessu   ✅Failedu   ❌u   🤷z Eval: u   🧠 New Memory: u   ⏳ Task Progress: 
u   📋 Future Plans: 
u   🤔 Thought: u   🎯 Summary: u   🛠️  Action r#   /z: Texclude_unsetN)current_stateprev_action_evaluationloggerinfoimportant_contentstask_progressfuture_plansthoughtsummary	enumerateactionlenmodel_dump_json)rj   rw   emojiir   s        ro   _log_responsezCustomAgent._log_response   s\   ..EEEE//FFFEEugWX%;%;%R%R$STU'(>(>(Q(Q'RST+H,B,B,P,P+QRS+H,B,B,O,O+PQRnX%;%;%C%C$DEFnX%;%;%C%C$DEF"8??3 	IAvKK"1q5'3x+?*@6CYCYhlCYCmBno	rp   model_output	step_infoc                    |y|xj                   dz  c_         |j                  j                  }|r*d|vr&||j                  vr|xj                  |dz   z  c_        |j                  j                  }|rd|vr||_        |j                  j
                  }|rd|vr||_        t        j                  d|j                          y)z"
        update step info
        Nr#   None
u   🧠 All Memory: 
)step_numberr~   r   memoryr   r   r   r   )rj   r   r   r   r   r   s         ro   update_step_infozCustomAgent.update_step_info   s     ")77JJ""44&i.>.>> 2T 99$22@@V=8&3I##11>>F,6%1I"))*:*:);<=rp   z--get_next_actioninput_messagesc           
      $	  K   | j                   j                  |      }| j                  j                  |       t	        |d      rIt
        j                  d       t
        j                  |j                         t
        j                  d       t        |j                  t              r|j                  d   }n|j                  }t
        j                  d|dd  d       |j                  d	d
      j                  dd
      }d}t        j                  ||      }|r/t        |t              }t
        j!                  d|dd  d       |}	 t#        |      }t
        j!                  d|dd  d       	 t+        j.                  |      }	t        |	t0              sLt
        j3                  dt5        |	      j6                          t9        dt5        |	      j6                         d|	vr t
        j3                  d       t9        d      d|	vr t
        j3                  d       t9        d      t        |	d   t0              s:t
        j3                  d t5        |	d         j6                          d!d
d
d"d#d$d|	d<   t        |	d   t              s3t
        j3                  d%t5        |	d         j6                          g |	d<   	  | j:                  d0i |	}
|
j<                  d| j>                   |
_        | jA                  |
       | xjB                  d/z  c_!        |
S # t$        $ rW}t
        j'                  dt)        |              t+        j,                  dt)        |       d
dddddg d      }Y d}~d}~ww xY w# t$        $ rv}t
        j'                  d&t)        |              d|	v r4g d}|D ]+  }||	d   vst
        j3                  d'| d(       d
|	d   |<   -  | j:                  d0i |	}
Y d}~#d}~ww xY w# t$        $ r}t
        j'                  d)t)        |              t
        j!                  d*|        t
        j!                  d+|        d,t)        |       d
ddd-d.dg d} | j:                  d0i |}
Y d}~d}~ww xY ww)1z/Get next action from LLM based on current statereasoning_contentu   🤯 Start Deep Thinking: u   🤯 End Deep Thinkingr   z$Raw model output (first 200 chars): N   z...```jsonr*   ```z\{[\s\S]*\})rl   zFound potential JSON object: zRepaired JSON: zJSON repair failed: zFailed - JSON repair error: z Error in processing model outputzRetry with proper formattingz5The model generated output that could not be repairedzJSON repair failure)r   r   r   r   r   r   )r~   r   z#Repaired JSON is not a dictionary: zExpected a dictionary, got r~   z(Missing 'current_state' in repaired JSONz(Missing 'current_state' in JSON responser   z!Missing 'action' in repaired JSONz!Missing 'action' in JSON responsez%'current_state' is not a dictionary: z%Failed - Invalid current_state formatz!Retry with proper JSON formattingz6The model generated an invalid current_state structurezInvalid current_state formatz'action' is not a list: zValidation error: zMissing required field 'z' in current_statezError parsing JSON response: zRaw content: zRepaired content: zFailed - JSON parsing error: z The model generated invalid JSONzJSON parsing failurer#    )"r7   invokeri   _add_message_with_tokensrc   r   r   r   
isinstancecontentlistr_   refindallmaxr   debugr$   	Exceptionerrorstrjsondumpsloadsdictwarningr.   __name__
ValueErrorr   r   rL   r   n_steps)rj   r   
ai_message
ai_contentjson_patternmatchespotential_jsonrepaired_contenteparsed_jsonparsedvalidation_errorrequired_fieldsfieldminimal_jsons                  ro   get_next_actionzCustomAgent.get_next_action   su     XX__^4
55jA:23KK45KK
445KK01j(($/#++A.J#++J 	::ds;K:LCPQ  ''	26>>ubI
 &**\:6 c2NLL89M8NcRS'J	3J?LL?+;DS+A*B#FG F	C**%56K k40!DT+EVE_E_D`ab #>tK?P?Y?Y>Z![\\k1IJ !KLL{*BC !DEE k/:DA!FtKXgLhGiGrGrFstu.U*,%'$GW=0O, k(3T:!9${8?T:U:^:^9_`a(*H%F&6d&6&6&E&EJ &A(A(AB6"y  	LL/Ax89#zz0LSQRVH.U*,%G$BV4" 
+ 
 	h  F1#6F2G1HIJ"k1']O!0 E O(DD"NN-EeWL^+_`BDK8?E '7d&6&6&E&EF  	CLL8QABLL=56LL-.>-?@A
 1NcRSfX.V*,%G$BA5" 
L #3$"2"2"B\"BF%	Cs   E	R'L 4EO> M< AR	M9"AM4.R4M99R<	O;5O6;5O60O> 6O;;O> >	RA;RRRRc                 *  K   | j                   syt        | j                        j                         g| j                  j                         dd }| j                  st| j                  rh|d   }d}t        |j                  t              r,|j                  D ]  }|d   dk(  r	||d   z  }|d   dk(  s n|j                  }t        |      |d<   | j                   j                  |       d{   }|j                  }|d   }t        |j                  t              r,|j                  D ]  }|d   dk(  s|dxx   d	| d
z  cc<    n|xj                  d	| dz  c_	        	 t        j                  |j                  dd      j                  dd            }t         j#                  dt        j$                  |d              t'        |d      rJt         j#                  d       t         j#                  |j(                         t         j#                  d       yy7 /# t        j*                  $ r t         j#                  d|        Y yt,        $ r:}t         j/                  d|        t         j#                  d|        Y d}~yd}~ww xY ww)z7Run the planner to analyze state and suggest next stepsNr#   r*   r.   text	image_url)r   z 
Planning Agent outputs plans:
 r   z
 r   r   u   📋 Plans:
   )indentr   u#   🤯 Start Planning Deep Thinking: u   🤯 End Planning Deep Thinkingz!Error parsing planning analysis: u   📋 Plans: )rT   r    rZ   get_system_messageri   get_messagesr=   r<   r   r   r   r   ainvoker   r   r_   r   r   r   rc   r   JSONDecodeErrorr   r   )	rj   planner_messageslast_state_messagenew_msgmsgrw   plan	plan_jsonr   s	            ro   _run_plannerzCustomAgent._run_plannero  sd      $223FFH
!!..04

 **t!1"!5G,44d;-55 !C6{f,3v;.V3 	! -44#/#@R  ))112BCC-b1(00$7)11 Qv;&(K%GvR#PPKQ &&,NtfTW*XX&	/

4<<	2#>#F#Fub#QRIKK-

9Q(G'HIJx!45ABH667=> 6 D( ## 	0KK-v./ 	/LL<QC@AKK,tf-..	/sV   B0J3<J/H0AJ7-J%B7H! J!+JJJ0JJJJz--stepc                 
   K   t         j                  d j                          d}d}g }g }	  j                  j	                          d{   } j                           j                  j                  | j                   j                  | j                          j                  r4 j                   j                  z  dk(  r j                          d{     j                  j                         } j                          	  j                  |       d{   } j                   r j!                  || j                          j#                  ||        j%                  ||        j&                  dk7  r j                  j)                  d        j                          |j,                  } j.                  j1                  | j                   j2                   j4                   fd j6                         d{   }t9        |      t9        |      k7  rxt;        t9        |      t9        |            D ]W  }|j=                  t?        dd||   jA                  d	       d
|t9        |      dz
     jA                  d	       d             Y |D ]=  }	|	jB                  sd|	jB                  v s xjB                  |	jB                  z  c_!        ? | _	        | _        t9        |      dkD  re|d   jD                  rV jB                  s|jF                   _!         jB                  |d   _!        t         j                  d|d   jB                          d _$        |r*|j,                  D 
cg c]  }
|
jM                  d	       c}
ng } jN                  jQ                  tS         jT                   j                  | jH                  |r'|D cg c]  }|jV                  s|jV                   c}ndg             |sy|r jY                  |||       yy7 7 E7 # t*        $ r"} j                  j)                  d       |d}~ww xY w7 N# t*        $ r,} jK                  |       d{  7  }| _	        Y d}~#d}~ww xY wc c}
w c c}w # |r0|j,                  D 
cg c]  }
|
jM                  d	       nc c}
w c}
ng } jN                  jQ                  tS         jT                   j                  | jH                  |r-|D cg c]  }|jV                  s|jV                   nc c}w c}ndg             |s Y y|r jY                  |||       w w xY ww)zExecute one step of the tasku   
📍 Step Nr   rY   r   c                  $     j                         S N_check_if_stopped_or_pausedru   s   ro   <lambda>z"CustomAgent.step.<locals>.<lambda>  s    d.N.N.P rp   )rS   rH   check_break_if_pausedrI   Tr|   zo is Failed to execute.                                                     Something new appeared after action r#   F)rf   include_in_memoryr   is_donezExtracted pageu   📄 Result: z	No result)agent_idstepactionsconsecutive_failures
step_error)-r   r   r   r:   	get_stater   ri   add_state_messagere   _last_resultr<   rT   planning_intervalr   r   r   rO   r   _save_conversationrX   _remove_state_message_by_indexr   r   r;   	multi_actrS   rH   rI   r   rangeappendr   r   rf   r   r   r   _handle_step_error
model_dump	telemetrycapturer   r   r   _make_history_item)rj   r   stater   resultr   r   r   riret_ars   `           ro   r   zCustomAgent.step  s     	l4<<.12%'%'M	E..88::E,,.  225$:L:LdN_N_aj37??D DLL43I3I$IQ$N'')))!11>>@N,,.%)%9%9.%II2233E<V%%lI>''E??&99((GGK002 *6)<)<G/3/H/H$$$($<$<#22&P%)%>%> 0I 0 *F 6{c'l*FS\: ?BMM,AE8?8S8Sbf8S8g7h iYY`adekalopapYq  ZB  ZB  QU  ZB  ZV  YW6X7<	#> ??  E)).>$BXBX.X**d.D.DD*E !'D!(D6{Q6":#5#5---6-=-=D*/3/E/Er
,mF2J,H,H+IJK()D% ZfATATUAq||$|7UkmGNN""'!]]#)-)B)BGMCA177CT_S` ''eVD W ; *  J  $$CCBG*<  	'221555F &D	'
 V  D ZfATATUAq||$|7UUkmGNN""'!]]#)-)B)BGMCA177CCT_S` ''eVD s  +U"Q PBQ P.Q 	P$ P!BP$ )AQ QB$Q *Q 9B)Q "U"2RAU"R$R2)U"Q Q !P$ $	Q-Q

QQ 	R
R2Q53R?R R

R 
U"U(SAUT/ T/.UU"UU"	max_stepsc                 x   K   	  j                           j                  r[ j                  j                   j                   j                  d j
                   fd j                         d{   }| _        t         j                   j                  d|ddd      }t        |      D ]  } j                         r n j                  |       d{     j                  j                         sJ j                   r!||dz
  k  r j#                          d{   swt$        j'                  d        n t$        j'                  d	        j(                  s6|j*                   j                  j                  d
   j,                  d
   _        n5 j(                   j                  j                  d
   j,                  d
   _         j                   j.                  j1                  t3         j4                   j                  j                          j6                   j6                  |k\   j                  j9                                       j:                  s" j                  j=                          d{     j>                  s. j@                  r" j@                  j=                          d{     jB                  r;d}tE         jB                  tF              r jB                  } jI                  |       S S 7 7 )7 7 7 W#  j.                  j1                  t3         j4                   j                  j                          j6                   j6                  |k\   j                  j9                                       j:                  s# j                  j=                          d{  7    j>                  s/ j@                  r# j@                  j=                          d{  7    jB                  r;d}tE         jB                  tF              r jB                  } jI                  |       w w xY ww)z-Execute the task with maximum number of stepsFc                  $     j                         S r   r   ru   s   ro   r   z!CustomAgent.run.<locals>.<lambda>	  s    $2R2R2T rp   )check_for_new_elementsrS   r   rI   Nr#   r*   )r6   r8   r   r   r   r   r   u   ✅ Task completed successfullyu,   ❌ Failed to complete task in maximum stepsr   )r   successstepsmax_steps_reachederrorsagent_history.gif)output_path)%_log_agent_runrN   r;   r   r:   rS   rI   r   r'   r6   r8   r   _too_many_failuresr   historyr   rE   _validate_outputr   r   rf   r   r   r   r   r   r   r   r   injected_browser_contextcloseinjected_browserr9   rG   r   r   create_history_gif)rj   r   r   r   r   r   s   `     ro   runzCustomAgent.run  sj    H	A! ###88((((+0(,(@(@*T)-)B)B  9    %+!+YY..# I i( c**, ii	***<<'') 00TIM5I%)%:%:%<<<$KK ABc" JK--LUL\L\DLL((,33B7ILPLbLbDLL((,33B7I<< NN""&!]] LL002,,&*lli&?<<..0 00**00222((T\\ll((***  #6d//5"&"3"3K''K'@ !}2 +  =4 3 + NN""&!]] LL002,,&*lli&?<<..0 00**00222((T\\ll((***  #6d//5"&"3"3K''K'@ !s   P:A+L
 0K=1A&L
 L L
 7'L
 LB4L
 B#P:6L79P:0L1AP:=L
  L
 L
 P:P:
B$P7.N1/:P7)O,*AP77P:r   duration
show_goals	show_task	show_logo	font_sizetitle_font_sizegoal_font_sizemarginline_spacingc           
      z   | j                   j                   st        j                  d       yg }| j                   j                   r-| j                   j                   d   j                  j                  st        j                  d       y	 g d}d}|D ]  }	 t        j                         dk(  r7t        j                  j                  t        j                  dd	      |d
z         }t        j                  ||      }t        j                  ||      }t        j                  ||      }d} n |st        d      	 d}|rk	 t        j                   d      }d}|j"                  |j$                  z  }t'        ||z        }|j)                  ||ft        j*                  j,                        }|rh| j0                  r\| j3                  | j0                  | j                   j                   d   j                  j                  ||
      }|j5                  |       t7        | j                   j                   d      D ]  \  }}|j                  j                  st9        j:                  |j                  j                        }t        j                   t=        j>                  |            }|rB|j@                  r6| jC                  |||j@                  jD                  jF                  |	|      }|j5                  |        |r6|d   jI                  |d|dd |dd       t        jK                  d|        yt        j                  d       y# t        $ r Y w xY w# t        $ r. t        j                         }t        j                         }|}Y Rw xY w# t.        $ r#}t        j                  d|        Y d}~d}~ww xY w)zGCreate a GIF from the agent's history with overlaid task and goal text.zNo history to create GIF fromNr   z1No history or first screenshot to create GIF from)	HelveticaArial
DejaVuSansVerdanaFWindowsWIN_FONT_DIRzC:\Windows\Fontsz.ttfTzNo preferred fonts foundz./static/browser-use.png   zCould not load logo: r#   )imager   	goal_textregular_font
title_fontr  logo)save_allappend_imagesr  loopoptimizezCreated GIF at z(No images found in history to create GIF)&r   r   r   r   
screenshotplatformsystemr[   pathjoingetenvr   truetypeOSErrorload_defaultr	   openwidthheightintresize
ResamplingLANCZOSr   r6   _create_task_framer   r   base64	b64decodeioBytesIOr   _add_overlay_to_imager~   r   saver   )rj   r   r  r  r  r  r	  r
  r  r  r  imagesfont_optionsfont_loaded	font_namer  r  	goal_fontr  logo_heightaspect_ratio
logo_widthr   
task_framer   itemimg_datar  s                               ro   r  zCustomAgent.create_history_gifI  sL    ||##NN:;||##4<<+?+?+B+H+H+S+SNNNO	%JLK) 	
(I5$&GGLL>K_1`bkntbt$u	#,#5#5i#KL!*!3!3I!OJ ) 2 29n MI"&K 899  <zz"<=!#zzDKK7 |!;<
{{J#<e>N>N>V>VW
 00		$$Q'--88J MM*% !!5!5q9 	!GAt::(( ''

(=(=>HJJrzz(34Ed//22 !"//==EE!-)! 3  MM% '	!* 1INN$QRj!   KK/+78NNEFO    	%$113L"//1J$I		%   <!6qc:;;<sP   	M BM'M =A*N 	MM MM 3N
N	N:N55N:)rq   Nr   )d   )
r   i  TTF(   8   ,   rB  g      ?)$r   
__module____qualname__r   r   r   r   r   r   r   boolr   r+  r   r   r   r   r   r   rb   rv   r&   r   r'   r   r   r   r   r   r   r   r   r  floatr  __classcell__)rn   s   @ro   r)   r)   2   s   
  &*59%/\#+048=D !!6B;M$*$)-1'+7;8<- %((*)-IMfjRV17;?37$%Y

 
 	

 t^
 ,d2
 #
 
 %)
 %-SM
 .6c]
 
 
 "&l!3
 !%%7 8
  "!
" "#
$ &c]%
& *'
( %T#s(^4)
* #+49"5+
, !%S	-
D "E
F #&G
H #'I
J &d4T#s(^0C+D&EFK
N )1.-QT1UW[1[(\_c(cO
P %-.@-A4-G$H4$OQ
R "*#S
T "*-!8U
V "-0W
X "Y
BX&7 D * UY> 1>>Q>8 -.AD4E A+ A /AF4/HSM 4/l (#UEH-@$A UET UE $UEnJA3 JA1A JA\  3 #"##%"$"%oGoG oG 	oG
 oG oG oG !oG  oG oG  oG 
oGrp   r)   )Cr   loggingpdb	tracebacktypingr   r   r   r   r   r   PILr	   r
   r   r[   r0  r2  r   browser_use.agent.promptsr   r   browser_use.agent.servicer   browser_use.agent.viewsr   r   r   r   r   browser_use.browser.browserr   browser_use.browser.contextr   browser_use.browser.viewsr   browser_use.controller.servicer   browser_use.telemetry.viewsr   r   r   browser_use.utilsr   *langchain_core.language_models.chat_modelsr   langchain_core.messagesr   r   r   r    json_repairr!   src.utils.agent_stater"   custom_json_repairr$   custom_message_managerr%   custom_viewsr&   r'   r   	getLoggerr   r   r)   r   rp   ro   <module>r`     s      
  < < + + 	  	  F +  0 6 9 5 
 3 D 
 4 # , 4 8 @ 				8	$F
G% F
Grp   