Finetuned model doesnot support tools in ollama

[DeekshithaDPrakash]

I have a Finetuned llama model for question answering task saved as modelsafetensors.

I was trying to use this as an agent to answer "yes" or "no". Although I have created the ollama model for this, during the code execution to it throws an error:

class GradeAnswer(BaseModel):
    """Binary score to assess answer addresses question."""

    binary_score: str = Field(
        description="Answer addresses the question, 'yes' or 'no'"
    )

# ollama  -------------------------------------------------------------------

llm = ChatOllama(model="llama32_3B_Finetuned", temperature=0, max_tokens=4096)

structured_llm_grader = llm.with_structured_output(GradeAnswer)

system = """You are a grader assessing whether an answer addresses / resolves a question \n 
     Give a binary score 'yes' or 'no'. Yes' means that the answer resolves the question."""


answer_prompt = ChatPromptTemplate.from_messages(
    [
        ("system", system),
        ("human", "User question: \n\n {question} \n\n LLM generation: {generation}"),
    ]
)

answer_grader = answer_prompt | structured_llm_grader
answer_grader.invoke({"question": question, "generation": generation})

Error:

ResponseError                             Traceback (most recent call last)
Cell In[78], line 33
     30 question_router = route_prompt | structured_llm_router
     32 # Invoke the chain
---> 33 response = question_router.invoke({"question": "소형 무인 비행기"})
     34 print("Parsed Structured Output:", response)
     35 print(f"Datasource: {response.datasource}")

File ~/miniconda3/envs/jupyter_env/lib/python3.10/site-packages/langchain_core/runnables/base.py:3024, in RunnableSequence.invoke(self, input, config, **kwargs)
   3022             input = context.run(step.invoke, input, config, **kwargs)
   3023         else:
-> 3024             input = context.run(step.invoke, input, config)
   3025 # finish the root run
   3026 except BaseException as e:

File ~/miniconda3/envs/jupyter_env/lib/python3.10/site-packages/langchain_core/runnables/base.py:5354, in RunnableBindingBase.invoke(self, input, config, **kwargs)
   5348 def invoke(
   5349     self,
   5350     input: Input,
   5351     config: Optional[RunnableConfig] = None,
   5352     **kwargs: Optional[Any],
   5353 ) -> Output:
-> 5354     return self.bound.invoke(
   5355         input,
   5356         self._merge_configs(config),
   5357         **{**self.kwargs, **kwargs},
   5358     )

File ~/miniconda3/envs/jupyter_env/lib/python3.10/site-packages/langchain_core/language_models/chat_models.py:286, in BaseChatModel.invoke(self, input, config, stop, **kwargs)
    275 def invoke(
    276     self,
    277     input: LanguageModelInput,
   (...)
    281     **kwargs: Any,
    282 ) -> BaseMessage:
    283     config = ensure_config(config)
    284     return cast(
    285         ChatGeneration,
--> 286         self.generate_prompt(
    287             [self._convert_input(input)],
    288             stop=stop,
    289             callbacks=config.get("callbacks"),
    290             tags=config.get("tags"),
    291             metadata=config.get("metadata"),
    292             run_name=config.get("run_name"),
    293             run_id=config.pop("run_id", None),
    294             **kwargs,
    295         ).generations[0][0],
    296     ).message

File ~/miniconda3/envs/jupyter_env/lib/python3.10/site-packages/langchain_core/language_models/chat_models.py:786, in BaseChatModel.generate_prompt(self, prompts, stop, callbacks, **kwargs)
    778 def generate_prompt(
    779     self,
    780     prompts: list[PromptValue],
   (...)
    783     **kwargs: Any,
    784 ) -> LLMResult:
    785     prompt_messages = [p.to_messages() for p in prompts]
--> 786     return self.generate(prompt_messages, stop=stop, callbacks=callbacks, **kwargs)

File ~/miniconda3/envs/jupyter_env/lib/python3.10/site-packages/langchain_core/language_models/chat_models.py:643, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, run_id, **kwargs)
    641         if run_managers:
    642             run_managers[i].on_llm_error(e, response=LLMResult(generations=[]))
--> 643         raise e
    644 flattened_outputs = [
    645     LLMResult(generations=[res.generations], llm_output=res.llm_output)  # type: ignore[list-item]
    646     for res in results
    647 ]
    648 llm_output = self._combine_llm_outputs([res.llm_output for res in results])

File ~/miniconda3/envs/jupyter_env/lib/python3.10/site-packages/langchain_core/language_models/chat_models.py:633, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, run_id, **kwargs)
    630 for i, m in enumerate(messages):
    631     try:
    632         results.append(
--> 633             self._generate_with_cache(
    634                 m,
    635                 stop=stop,
    636                 run_manager=run_managers[i] if run_managers else None,
    637                 **kwargs,
    638             )
    639         )
    640     except BaseException as e:
    641         if run_managers:

File ~/miniconda3/envs/jupyter_env/lib/python3.10/site-packages/langchain_core/language_models/chat_models.py:851, in BaseChatModel._generate_with_cache(self, messages, stop, run_manager, **kwargs)
    849 else:
    850     if inspect.signature(self._generate).parameters.get("run_manager"):
--> 851         result = self._generate(
    852             messages, stop=stop, run_manager=run_manager, **kwargs
    853         )
    854     else:
    855         result = self._generate(messages, stop=stop, **kwargs)

File ~/miniconda3/envs/jupyter_env/lib/python3.10/site-packages/langchain_ollama/chat_models.py:690, in ChatOllama._generate(self, messages, stop, run_manager, **kwargs)
    683 def _generate(
    684     self,
    685     messages: List[BaseMessage],
   (...)
    688     **kwargs: Any,
    689 ) -> ChatResult:
--> 690     final_chunk = self._chat_stream_with_aggregation(
    691         messages, stop, run_manager, verbose=self.verbose, **kwargs
    692     )
    693     generation_info = final_chunk.generation_info
    694     chat_generation = ChatGeneration(
    695         message=AIMessage(
    696             content=final_chunk.text,
   (...)
    700         generation_info=generation_info,
    701     )

File ~/miniconda3/envs/jupyter_env/lib/python3.10/site-packages/langchain_ollama/chat_models.py:591, in ChatOllama._chat_stream_with_aggregation(self, messages, stop, run_manager, verbose, **kwargs)
    582 def _chat_stream_with_aggregation(
    583     self,
    584     messages: List[BaseMessage],
   (...)
    588     **kwargs: Any,
    589 ) -> ChatGenerationChunk:
    590     final_chunk = None
--> 591     for stream_resp in self._create_chat_stream(messages, stop, **kwargs):
    592         if not isinstance(stream_resp, str):
    593             chunk = ChatGenerationChunk(
    594                 message=AIMessageChunk(
    595                     content=(
   (...)
    608                 ),
    609             )

File ~/miniconda3/envs/jupyter_env/lib/python3.10/site-packages/langchain_ollama/chat_models.py:580, in ChatOllama._create_chat_stream(self, messages, stop, **kwargs)
    578     yield from self._client.chat(**chat_params)
    579 else:
--> 580     yield self._client.chat(**chat_params)

File ~/miniconda3/envs/jupyter_env/lib/python3.10/site-packages/ollama/_client.py:332, in Client.chat(self, model, messages, tools, stream, format, options, keep_alive)
    288 def chat(
    289   self,
    290   model: str = '',
   (...)
    297   keep_alive: Optional[Union[float, str]] = None,
    298 ) -> Union[ChatResponse, Iterator[ChatResponse]]:
    299   """
    300   Create a chat response using the requested model.
    301 
   (...)
    330   Returns `ChatResponse` if `stream` is `False`, otherwise returns a `ChatResponse` generator.
    331   """
--> 332   return self._request(
    333     ChatResponse,
    334     'POST',
    335     '/api/chat',
    336     json=ChatRequest(
    337       model=model,
    338       messages=[message for message in _copy_messages(messages)],
    339       tools=[tool for tool in _copy_tools(tools)],
    340       stream=stream,
    341       format=format,
    342       options=options,
    343       keep_alive=keep_alive,
    344     ).model_dump(exclude_none=True),
    345     stream=stream,
    346   )

File ~/miniconda3/envs/jupyter_env/lib/python3.10/site-packages/ollama/_client.py:177, in Client._request(self, cls, stream, *args, **kwargs)
    173         yield cls(**part)
    175   return inner()
--> 177 return cls(**self._request_raw(*args, **kwargs).json())

File ~/miniconda3/envs/jupyter_env/lib/python3.10/site-packages/ollama/_client.py:122, in Client._request_raw(self, *args, **kwargs)
    120   r.raise_for_status()
    121 except httpx.HTTPStatusError as e:
--> 122   raise ResponseError(e.response.text, e.response.status_code) from None
    123 return r

ResponseError: llama32_3B_Finetuned does not support tools

[SergioRubio01]

@DeekshithaDPrakash were you able to solve this issue?