Here is the full error trace for running the last cell (just running the unmodified nb e2e):
/home/jovyan/.local/lib/python3.11/site-packages/google/adk/models/llm_request.py:273: UserWarning: [EXPERIMENTAL] feature FeatureName.JSON_SCHEMA_FOR_FUNC_DECL is enabled.
declaration = tool._get_declaration()
Node execution failed with exception
Traceback (most recent call last):
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/workflow/_node_runner.py”, line 136, in run
await self._execute_node(ctx, node_input)
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/workflow/_node_runner.py”, line 274, in _execute_node
await self._run_node_loop(ctx, node_input)
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/workflow/_node_runner.py”, line 288, in _run_node_loop
async for event in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/workflow/_base_node.py”, line 166, in run
async for item in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/agents/llm_agent.py”, line 599, in _run_impl
async for event in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/workflow/_llm_agent_wrapper.py”, line 370, in run_llm_agent_as_node
async for event in run_iter:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/agents/base_agent.py”, line 306, in run_async
async for event in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/agents/llm_agent.py”, line 548, in _run_async_impl
async for event in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py”, line 956, in run_async
async for event in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py”, line 1036, in _run_one_step_async
async for llm_response in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py”, line 1483, in _call_llm_async
async for event in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py”, line 1461, in _call_llm_with_tracing
async for llm_response in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py”, line 1544, in _run_and_handle_error
async for response in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py”, line 419, in _run_and_handle_error
raise model_error
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py”, line 396, in _run_and_handle_error
async for llm_response in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/models/google_llm.py”, line 282, in generate_content_async
response = await self.api_client.aio.models.generate_content(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/google/genai/models.py”, line 8788, in generate_content
return await self._generate_content(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/google/genai/models.py”, line 7260, in _generate_content
response = await self._api_client.async_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/google/genai/_api_client.py”, line 1754, in async_request
result = await self._async_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/google/genai/_api_client.py”, line 1687, in _async_request
return await self._async_retry( # type: ignore[no-any-return]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/tenacity/asyncio/_init_.py”, line 112, in _call_
do = await self.iter(retry_state=retry_state)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/tenacity/asyncio/_init_.py”, line 157, in iter
result = await action(retry_state)
^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/tenacity/_utils.py”, line 111, in inner
return call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/tenacity/_init_.py”, line 413, in exc_check
raise retry_exc.reraise()
^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/tenacity/_init_.py”, line 184, in reraise
raise self.last_attempt.result()
^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/local/lib/python3.11/concurrent/futures/_base.py”, line 449, in result
return self.__get_result()
^^^^^^^^^^^^^^^^^^^
File “/usr/local/lib/python3.11/concurrent/futures/_base.py”, line 401, in __get_result
raise self._exception
File “/home/jovyan/.local/lib/python3.11/site-packages/tenacity/asyncio/_init_.py”, line 116, in _call_
result = await fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/google/genai/_api_client.py”, line 1607, in _async_request_once
response = await session.request( # type: ignore[union-attr]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/aiohttp/client.py”, line 858, in _request
resp = await handler(req)
^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/aiohttp/client.py”, line 810, in _connect_and_send_request
assert self._connector is not None
^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError
Root node image_agent failed.
Traceback (most recent call last):
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/runners.py”, line 601, in _drive_root_node
await root_ctx._run_node_internal(
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/agents/context.py”, line 603, in _run_node_internal
raise DynamicNodeFailError(
google.adk.workflow._errors.DynamicNodeFailError: Dynamic node image_agent failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/runners.py”, line 906, in _cleanup_root_task
await task
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/runners.py”, line 610, in _drive_root_node
raise e.error
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/workflow/_node_runner.py”, line 136, in run
await self._execute_node(ctx, node_input)
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/workflow/_node_runner.py”, line 274, in _execute_node
await self._run_node_loop(ctx, node_input)
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/workflow/_node_runner.py”, line 288, in _run_node_loop
async for event in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/workflow/_base_node.py”, line 166, in run
async for item in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/agents/llm_agent.py”, line 599, in _run_impl
async for event in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/workflow/_llm_agent_wrapper.py”, line 370, in run_llm_agent_as_node
async for event in run_iter:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/agents/base_agent.py”, line 306, in run_async
async for event in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/agents/llm_agent.py”, line 548, in _run_async_impl
async for event in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py”, line 956, in run_async
async for event in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py”, line 1036, in _run_one_step_async
async for llm_response in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py”, line 1483, in _call_llm_async
async for event in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py”, line 1461, in _call_llm_with_tracing
async for llm_response in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py”, line 1544, in _run_and_handle_error
async for response in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py”, line 419, in _run_and_handle_error
raise model_error
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py”, line 396, in _run_and_handle_error
async for llm_response in agen:
File “/home/jovyan/.local/lib/python3.11/site-packages/google/adk/models/google_llm.py”, line 282, in generate_content_async
response = await self.api_client.aio.models.generate_content(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/google/genai/models.py”, line 8788, in generate_content
return await self._generate_content(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/google/genai/models.py”, line 7260, in _generate_content
response = await self._api_client.async_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/google/genai/_api_client.py”, line 1754, in async_request
result = await self._async_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/google/genai/_api_client.py”, line 1687, in _async_request
return await self._async_retry( # type: ignore[no-any-return]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/tenacity/asyncio/_init_.py”, line 112, in _call_
do = await self.iter(retry_state=retry_state)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/tenacity/asyncio/_init_.py”, line 157, in iter
result = await action(retry_state)
^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/tenacity/_utils.py”, line 111, in inner
return call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/tenacity/_init_.py”, line 413, in exc_check
raise retry_exc.reraise()
^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/tenacity/_init_.py”, line 184, in reraise
raise self.last_attempt.result()
^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/local/lib/python3.11/concurrent/futures/_base.py”, line 449, in result
return self.__get_result()
^^^^^^^^^^^^^^^^^^^
File “/usr/local/lib/python3.11/concurrent/futures/_base.py”, line 401, in __get_result
raise self._exception
File “/home/jovyan/.local/lib/python3.11/site-packages/tenacity/asyncio/_init_.py”, line 116, in _call_
result = await fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/google/genai/_api_client.py”, line 1607, in _async_request_once
response = await session.request( # type: ignore[union-attr]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/aiohttp/client.py”, line 858, in _request
resp = await handler(req)
^^^^^^^^^^^^^^^^^^
File “/home/jovyan/.local/lib/python3.11/site-packages/aiohttp/client.py”, line 810, in _connect_and_send_request
assert self._connector is not None
^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError
---------------------------------------------------------------------------
DynamicNodeFailError Traceback (most recent call last)
File ~/.local/lib/python3.11/site-packages/google/adk/runners.py:601, in Runner._run_node_async.<locals>._drive_root_node()
600 try:
--> 601 await root_ctx._run_node_internal(
602 root_agent,
603 node_input=node_input,
604 resume_inputs=resume_inputs,
605 )
606 except NodeInterruptedError:
607 # The node was interrupted (e.g. for HITL).
File ~/.local/lib/python3.11/site-packages/google/adk/agents/context.py:603, in Context._run_node_internal(self, node, node_input, use_as_output, run_id, use_sub_branch, override_branch, override_isolation_scope, raise_on_wait, return_ctx, resume_inputs, skip_run_id_validation)
601 from ..workflow._errors import DynamicNodeFailError
--> 603 raise DynamicNodeFailError(
604 message=f'Dynamic node {curr_node.name} failed',
605 error=child_ctx.error,
606 error_node_path=child_ctx.error_node_path,
607 )
608 if child_ctx.interrupt_ids:
DynamicNodeFailError: Dynamic node image_agent failed
During handling of the above exception, another exception occurred:
AssertionError Traceback (most recent call last)
Cell In[16], line 14
10 role="user",
11 parts=[types.Part(text=prompt)]
12 )
13
---> 14 async for event in runner.run_async(
15 user_id="user",
16 session_id=session.id,
17 new_message=user_message,
File ~/.local/lib/python3.11/site-packages/google/adk/runners.py:1111, in Runner.run_async(self, user_id, session_id, invocation_id, new_message, state_delta, run_config, yield_user_message)
1094 raise ValueError(
1095 "LlmAgent as root agent must have mode='chat', but got"
1096 f" mode='{self.agent.mode}'."
1097 )
1098 async with aclosing(
1099 self._run_node_async(
1100 user_id=user_id,
(...) 1109 )
1110 ) as agen:
-> 1111 async for event in agen:
1112 yield event
1113 return
File ~/.local/lib/python3.11/site-packages/google/adk/runners.py:628, in Runner._run_node_async(self, user_id, session_id, invocation_id, new_message, state_delta, run_config, yield_user_message, node, session)
624 yield event
625 finally:
626 # _cleanup_root_task re-raises a root-node Exception (if any) after
627 # the event stream has drained.
--> 628 await self._cleanup_root_task(task, self.agent.name)
629 except Exception as e:
630 # An unhandled exception escaped runner execution. Notify plugins
631 # (notification-only) and re-raise. after_run stays success-only.
632 run_error = e
File ~/.local/lib/python3.11/site-packages/google/adk/runners.py:906, in Runner._cleanup_root_task(self, task, node_name)
904 task.cancel()
905 try:
--> 906 await task
907 except asyncio.CancelledError:
908 logger.warning('Root node %s was cancelled.', node_name)
File ~/.local/lib/python3.11/site-packages/google/adk/runners.py:610, in Runner._run_node_async.<locals>._drive_root_node()
608 pass
609 except DynamicNodeFailError as e:
--> 610 raise e.error
611 finally:
612 await ic._event_queue.put((done_sentinel, None))
File ~/.local/lib/python3.11/site-packages/google/adk/workflow/_node_runner.py:136, in NodeRunner.run(self, node_input, resume_inputs)
132 async with node_tracing.start_as_current_node_span(
133 self._parent_ctx, self._node
134 ) as telemetry_context:
135 ctx._telemetry_context = telemetry_context
--> 136 await self._execute_node(ctx, node_input)
137 await self._flush_output_and_deltas(ctx)
138 logger.debug("node %s end.", ctx.node_path)
File ~/.local/lib/python3.11/site-packages/google/adk/workflow/_node_runner.py:274, in NodeRunner._execute_node(self, ctx, node_input)
272 await self._run_node_loop_with_timeout(ctx, node_input, timeout)
273 else:
--> 274 await self._run_node_loop(ctx, node_input)
275 except NodeInterruptedError:
276 # A dynamic child interrupted via ctx.run_node().
277 # The child's interrupt_ids are already on ctx
278 # (set by the schedule callback). Nothing more to do —
279 # the caller reads ctx.interrupt_ids.
280 pass
File ~/.local/lib/python3.11/site-packages/google/adk/workflow/_node_runner.py:288, in NodeRunner._run_node_loop(self, ctx, node_input)
286 logger.debug("node %s execute loop start.", ctx.node_path)
287 async with Aclosing(self._node.run(ctx=ctx, node_input=node_input)) as agen:
--> 288 async for event in agen:
289 self._track_event_in_context(event, ctx)
290 await self._enqueue_event(event, ctx)
File ~/.local/lib/python3.11/site-packages/google/adk/workflow/_base_node.py:166, in BaseNode.run(self, ctx, node_input)
164 node_input = self._validate_input_data(node_input)
165 async with Aclosing(self._run_impl(ctx=ctx, node_input=node_input)) as agen:
--> 166 async for item in agen:
167 if item is None:
168 continue
File ~/.local/lib/python3.11/site-packages/google/adk/agents/llm_agent.py:599, in LlmAgent._run_impl(self, ctx, node_input)
594 from ..workflow._llm_agent_wrapper import run_llm_agent_as_node
596 async with Aclosing(
597 run_llm_agent_as_node(self, ctx=ctx, node_input=node_input)
598 ) as agen:
--> 599 async for event in agen:
600 # Keep the agent's true event author so the outer NodeRunner does
601 # not overwrite it with the parent workflow's event_author.
602 if event.author:
603 ctx.event_author = event.author
File ~/.local/lib/python3.11/site-packages/google/adk/workflow/_llm_agent_wrapper.py:370, in run_llm_agent_as_node(agent, ctx, node_input)
368 run_method = agent.run_live(ic) if is_live else agent.run_async(ic)
369 async with aclosing(run_method) as run_iter:
--> 370 async for event in run_iter:
371 yield event
372 task_fcs = _extract_task_delegation_fcs(event, tools_dict)
File ~/.local/lib/python3.11/site-packages/google/adk/agents/base_agent.py:306, in BaseAgent.run_async(self, parent_context)
303 return
305 async with Aclosing(self._run_async_impl(ctx)) as agen:
--> 306 async for event in agen:
307 yield event
309 if ctx.end_invocation:
File ~/.local/lib/python3.11/site-packages/google/adk/agents/llm_agent.py:548, in LlmAgent._run_async_impl(self, ctx)
546 output_accumulator = ''
547 async with Aclosing(self._llm_flow.run_async(ctx)) as agen:
--> 548 async for event in agen:
549 self.__maybe_save_output_to_state(event)
550 output_accumulator = self.__maybe_accumulate_streaming_output(
551 event, output_accumulator
552 )
File ~/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py:956, in BaseLlmFlow.run_async(self, invocation_context)
954 last_event = None
955 async with Aclosing(self._run_one_step_async(invocation_context)) as agen:
--> 956 async for event in agen:
957 last_event = event
958 yield event
File ~/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py:1036, in BaseLlmFlow._run_one_step_async(self, invocation_context)
1025 model_response_event = Event(
1026 id=Event.new_id(),
1027 invocation_id=invocation_context.invocation_id,
1028 author=invocation_context.agent.name,
1029 branch=invocation_context.branch,
1030 )
1031 async with Aclosing(
1032 self._call_llm_async(
1033 invocation_context, llm_request, model_response_event
1034 )
1035 ) as agen:
-> 1036 async for llm_response in agen:
1037 # Postprocess after calling the LLM.
1038 async with Aclosing(
1039 self._postprocess_async(
1040 invocation_context,
(...) 1044 )
1045 ) as agen:
1046 async for event in agen:
1047 # Partial chunks of one streaming response share the base id; mint a
1048 # fresh id only after a complete event so distinct responses differ.
File ~/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py:1483, in BaseLlmFlow._call_llm_async(self, invocation_context, llm_request, model_response_event)
1480 yield llm_response
1482 async with Aclosing(_call_llm_with_tracing()) as agen:
-> 1483 async for event in agen:
1484 yield event
File ~/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py:1461, in BaseLlmFlow._call_llm_async.<locals>._call_llm_with_tracing()
1447 responses_generator = llm.generate_content_async(
1448 llm_request,
1449 stream=invocation_context.run_config.streaming_mode
1450 == StreamingMode.SSE,
1451 )
1452 async with Aclosing(
1453 self._run_and_handle_error(
1454 responses_generator,
(...) 1459 )
1460 ) as agen:
-> 1461 async for llm_response in agen:
1462 trace_call_llm(
1463 invocation_context,
1464 model_response_event.id,
(...) 1467 span,
1468 )
1469 # Rebind to call_llm span for after_model_callback.
File ~/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py:1544, in BaseLlmFlow._run_and_handle_error(self, response_generator, invocation_context, llm_request, model_response_event, call_llm_span)
1527 async def _run_and_handle_error(
1528 self,
1529 response_generator: AsyncGenerator[LlmResponse, None],
(...) 1533 call_llm_span: Optional[trace.Span] = None,
1534 ) -> AsyncGenerator[LlmResponse, None]:
1535 async with Aclosing(
1536 _run_and_handle_error(
1537 response_generator,
(...) 1542 )
1543 ) as agen:
-> 1544 async for response in agen:
1545 yield response
File ~/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py:419, in _run_and_handle_error(response_generator, invocation_context, llm_request, model_response_event, call_llm_span)
417 yield error_response
418 else:
--> 419 raise model_error
File ~/.local/lib/python3.11/site-packages/google/adk/flows/llm_flows/base_llm_flow.py:396, in _run_and_handle_error(response_generator, invocation_context, llm_request, model_response_event, call_llm_span)
390 async with _instrumentation.record_inference_telemetry(
391 llm_request,
392 invocation_context,
393 model_response_event,
394 ) as tel_ctx:
395 async with Aclosing(response_generator) as agen:
--> 396 async for llm_response in agen:
397 tel_ctx.record_llm_response(invocation_context, llm_response)
398 yield llm_response
File ~/.local/lib/python3.11/site-packages/google/adk/models/google_llm.py:282, in Gemini.generate_content_async(self, llm_request, stream)
279 yield close_result
281 else:
--> 282 response = await self.api_client.aio.models.generate_content(
283 model=llm_request.model,
284 contents=llm_request.contents,
285 config=llm_request.config,
286 )
287 logger.info('Response received from the model.')
288 if logger.isEnabledFor(logging.DEBUG):
File ~/.local/lib/python3.11/site-packages/google/genai/models.py:8788, in AsyncModels.generate_content(self, model, contents, config)
8780 indices_str = ', '.join(map(str, incompatible_tools_indexes))
8781 logger.warning(
8782 'Tools at indices [%s] are not compatible with automatic function'
8783 ' calling (AFC). AFC is disabled. If AFC is intended, please'
(...) 8786 indices_str,
8787 )
-> 8788 return await self._generate_content(
8789 model=model, contents=contents, config=final_parsed_config
8790 )
8792 remaining_remote_calls_afc = _extra_utils.get_max_remote_calls_afc(
8793 final_parsed_config
8794 )
8795 logger.info(
8796 f'AFC is enabled with max remote calls: {remaining_remote_calls_afc}.'
8797 )
File ~/.local/lib/python3.11/site-packages/google/genai/models.py:7260, in AsyncModels._generate_content(self, model, contents, config)
7257 request_dict = _common.convert_to_dict(request_dict)
7258 request_dict = _common.encode_unserializable_types(request_dict)
-> 7260 response = await self._api_client.async_request(
7261 'post', path, request_dict, http_options
7262 )
7264 if config is not None and getattr(
7265 config, 'should_return_http_response', None
7266 ):
7267 return_value = types.GenerateContentResponse(sdk_http_response=response)
File ~/.local/lib/python3.11/site-packages/google/genai/_api_client.py:1754, in BaseApiClient.async_request(self, http_method, path, request_dict, http_options)
1743 async def async_request(
1744 self,
1745 http_method: str,
(...) 1748 http_options: Optional[HttpOptionsOrDict] = None,
1749 ) -> SdkHttpResponse:
1750 http_request = self._build_request(
1751 http_method, path, request_dict, http_options
1752 )
-> 1754 result = await self._async_request(
1755 http_request=http_request, http_options=http_options, stream=False
1756 )
1757 response_body = result.response_stream[0] if result.response_stream else ''
1758 return SdkHttpResponse(headers=result.headers, body=response_body)
File ~/.local/lib/python3.11/site-packages/google/genai/_api_client.py:1687, in BaseApiClient._async_request(self, http_request, http_options, stream)
1685 retry = tenacity.AsyncRetrying(**retry_kwargs)
1686 return await retry(self._async_request_once, http_request, stream) # type: ignore[no-any-return]
-> 1687 return await self._async_retry( # type: ignore[no-any-return]
1688 self._async_request_once, http_request, stream
1689 )
File ~/.local/lib/python3.11/site-packages/tenacity/asyncio/__init__.py:112, in AsyncRetrying.__call__(self, fn, *args, **kwargs)
110 is_async = _utils.is_coroutine_callable(fn)
111 while True:
--> 112 do = await self.iter(retry_state=retry_state)
113 if isinstance(do, DoAttempt):
114 try:
File ~/.local/lib/python3.11/site-packages/tenacity/asyncio/__init__.py:157, in AsyncRetrying.iter(self, retry_state)
155 result = None
156 for action in self.iter_state.actions:
--> 157 result = await action(retry_state)
158 return result
File ~/.local/lib/python3.11/site-packages/tenacity/_utils.py:111, in wrap_to_async_func.<locals>.inner(*args, **kwargs)
110 async def inner(*args: typing.Any, **kwargs: typing.Any) -> typing.Any:
--> 111 return call(*args, **kwargs)
File ~/.local/lib/python3.11/site-packages/tenacity/__init__.py:413, in BaseRetrying._post_stop_check_actions.<locals>.exc_check(rs)
411 retry_exc = self.retry_error_cls(fut)
412 if self.reraise:
--> 413 raise retry_exc.reraise()
414 raise retry_exc from fut.exception()
File ~/.local/lib/python3.11/site-packages/tenacity/__init__.py:184, in RetryError.reraise(self)
182 def reraise(self) -> t.NoReturn:
183 if self.last_attempt.failed:
--> 184 raise self.last_attempt.result()
185 raise self
File /usr/local/lib/python3.11/concurrent/futures/_base.py:449, in Future.result(self, timeout)
447 raise CancelledError()
448 elif self._state == FINISHED:
--> 449 return self.__get_result()
451 self._condition.wait(timeout)
453 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
File /usr/local/lib/python3.11/concurrent/futures/_base.py:401, in Future.__get_result(self)
399 if self._exception:
400 try:
--> 401 raise self._exception
402 finally:
403 # Break a reference cycle with the exception in self._exception
404 self = None
File ~/.local/lib/python3.11/site-packages/tenacity/asyncio/__init__.py:116, in AsyncRetrying.__call__(self, fn, *args, **kwargs)
114 try:
115 if is_async:
--> 116 result = await fn(*args, **kwargs)
117 else:
118 result = fn(*args, **kwargs)
File ~/.local/lib/python3.11/site-packages/google/genai/_api_client.py:1607, in BaseApiClient._async_request_once(self, http_request, stream)
1605 url = url.replace('googleapis.com', 'mtls.googleapis.com')
1606 try:
-> 1607 response = await session.request( # type: ignore[union-attr]
1608 method=http_request.method,
1609 url=url,
1610 headers=http_request.headers,
1611 data=data,
1612 timeout=aiohttp.ClientTimeout(total=http_request.timeout),
1613 **self._async_client_session_request_args,
1614 )
1615 await errors.APIError.raise_for_async_response(response)
1616 unwrapped_response: Any = response
File ~/.local/lib/python3.11/site-packages/aiohttp/client.py:858, in ClientSession._request(self, method, str_or_url, params, data, json, cookies, headers, skip_auto_headers, auth, allow_redirects, max_redirects, compress, chunked, expect100, raise_for_status, read_until_eof, proxy, proxy_auth, timeout, verify_ssl, fingerprint, ssl_context, ssl, server_hostname, proxy_headers, trace_request_ctx, read_bufsize, auto_decompress, max_line_size, max_field_size, max_headers, middlewares)
855 handler = _connect_and_send_request
857 try:
--> 858 resp = await handler(req)
859 # Client connector errors should not be retried
860 except (
861 ConnectionTimeoutError,
862 ClientConnectorError,
863 ClientConnectorCertificateError,
864 ClientConnectorSSLError,
865 ):
File ~/.local/lib/python3.11/site-packages/aiohttp/client.py:810, in ClientSession._request.<locals>._connect_and_send_request(req)
806 async def _connect_and_send_request(
807 req: ClientRequest,
808 ) -> ClientResponse:
809 # connection timeout
--> 810 assert self._connector is not None
811 try:
812 conn = await self._connector.connect(
813 req, traces=traces, timeout=real_timeout
814 )
AssertionError: