Use a cloud based model

lib/friendly_error_messages_inference_client.rb

@@ -3,17 +3,14 @@
 class FriendlyErrorMessagesInferenceClient
   class Error < StandardError; end
 
-  MAX_RESULT_LENGTH_CHARS = 500
+  MAX_COMPLETION_TOKENS = 300
 
-  SYSTEM_PROMPT_BASE = <<~PROMPT
+  SYSTEM_PROMPT = <<~PROMPT
     You help young people learn to program in Python.
 
     A learner will show you some Python code, and usually an error message. Respond with a
     short piece of feedback, addressed to them, that helps them find and fix the problem
     themselves.
-  PROMPT
-
-  SYSTEM_PROMPT_L1_EXPLAIN = <<~PROMPT
     The learner has only seen the raw Python error. Nothing has explained it to them yet.
 
     Say in plain language what the error means, and suggest which part of their code to
@@ -27,7 +24,7 @@ class Error < StandardError; end
 
   def initialize
     @client = OpenAI::Client.new(
-      base_url: ENV.fetch('FEM_INFERENCE_API_URL'),
+      base_url: "#{ENV.fetch('FEM_INFERENCE_API_URL', '')}/ai/v1",
       api_key: ENV.fetch('FEM_INFERENCE_API_KEY', ''),
       timeout: 15.0,
       max_retries: 0
@@ -37,21 +34,31 @@ def initialize
   def get_message(message, trace, runtime, code)
     return 'This is a placeholder friendly error message.' if ENV.fetch('BYPASS_FEM_INFERENCE', nil) == 'true'
 
-    result = @client.responses.create(
-      model: 'qwen3.5:9b',
-      input: [
-        { role: 'system', content: SYSTEM_PROMPT_BASE },
-        { role: 'system', content: SYSTEM_PROMPT_L1_EXPLAIN },
-        { role: 'user', content: JSON.dump({ message:, trace:, runtime:, code: }) }
-      ],
-      reasoning: { effort: :none }
-    ).output_text
+    result = @client.chat.completions.create(
+      build_request(message, trace, runtime, code)
+    ).choices.first.message.content.to_s
 
     raise Error, 'No message returned by inference service.' if result.empty?
-    raise Error, "The message by the inference service was too long - max #{MAX_RESULT_LENGTH_CHARS} chars, actual #{result.length} chars." if result.length > MAX_RESULT_LENGTH_CHARS
 
     result
   rescue OpenAI::Errors::APIError => e
     raise Error, e.message
   end
+
+  def build_request(message, trace, runtime, code)
+    {
+      model: '@cf/qwen/qwen3.8-27b',
+      messages: [
+        { role: 'system', content: SYSTEM_PROMPT },
+        { role: 'user', content: JSON.dump({ message:, trace:, runtime:, code: }) }
+      ],
+      temperature: 0,
+      max_completion_tokens: MAX_COMPLETION_TOKENS,
+
+      request_options: {
+        extra_headers: { 'cf-aig-gateway-id' => ENV.fetch('FEM_GATEWAY_NAME', '') },
+        extra_body: { chat_template_kwargs: { enable_thinking: false } }
+      }
+    }
+  end
 end

spec/lib/friendly_error_messages_inference_client_spec.rb

@@ -6,6 +6,15 @@
   subject(:client) { described_class.new }
 
   let(:mock_client) { instance_double(OpenAI::Client) }
+  let(:chat) { instance_double(OpenAI::Resources::Chat) }
+  let(:completions) { instance_double(OpenAI::Resources::Chat::Completions) }
+
+  def stub_completion(content)
+    message = instance_double(OpenAI::Models::Chat::ChatCompletionMessage, content:)
+    choice = instance_double(OpenAI::Models::Chat::ChatCompletion::Choice, message:)
+    completion = instance_double(OpenAI::Models::Chat::ChatCompletion, choices: [choice])
+    allow(completions).to receive(:create).and_return(completion)
+  end
 
   context 'when BYPASS_FEM_INFERENCE is false' do
     around do |example|
@@ -14,51 +23,33 @@
       end
     end
 
-    it 'calls the inference server' do
-      responses = instance_double(OpenAI::Resources::Responses)
-      response = instance_double(OpenAI::Models::Responses::Response, output_text: 'A friendly error message')
+    before do
       allow(OpenAI::Client).to receive(:new).and_return(mock_client)
-      allow(mock_client).to receive(:responses).and_return(responses)
-      allow(responses).to receive(:create).and_return(response)
+      allow(mock_client).to receive(:chat).and_return(chat)
+      allow(chat).to receive(:completions).and_return(completions)
+    end
+
+    it 'calls the inference server' do
+      stub_completion('A friendly error message')
 
       result = client.get_message('', '', '', '')
       expect(result).to eq 'A friendly error message'
     end
 
     it 'throws an error when the inference call times out' do
-      responses = instance_double(OpenAI::Resources::Responses)
-      allow(OpenAI::Client).to receive(:new).and_return(mock_client)
-      allow(mock_client).to receive(:responses).and_return(responses)
-      allow(responses).to receive(:create).and_raise(OpenAI::Errors::APITimeoutError.new(url: 'http://some_url'))
+      allow(completions).to receive(:create).and_raise(OpenAI::Errors::APITimeoutError.new(url: 'http://some_url'))
 
       expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error)
     end
 
     it 'throws an error when the inference server is down' do
-      responses = instance_double(OpenAI::Resources::Responses)
-      allow(OpenAI::Client).to receive(:new).and_return(mock_client)
-      allow(mock_client).to receive(:responses).and_return(responses)
-      allow(responses).to receive(:create).and_raise(OpenAI::Errors::APIConnectionError.new(url: 'http://some_url'))
+      allow(completions).to receive(:create).and_raise(OpenAI::Errors::APIConnectionError.new(url: 'http://some_url'))
 
       expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error)
     end
 
     it 'throws an error when the inference server returns nothing' do
-      responses = instance_double(OpenAI::Resources::Responses)
-      response = instance_double(OpenAI::Models::Responses::Response, output_text: '')
-      allow(OpenAI::Client).to receive(:new).and_return(mock_client)
-      allow(mock_client).to receive(:responses).and_return(responses)
-      allow(responses).to receive(:create).and_return(response)
-
-      expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error)
-    end
-
-    it 'throws an error when the inference server returns a response that is too long' do
-      responses = instance_double(OpenAI::Resources::Responses)
-      response = instance_double(OpenAI::Models::Responses::Response, output_text: 'a' * 1000)
-      allow(OpenAI::Client).to receive(:new).and_return(mock_client)
-      allow(mock_client).to receive(:responses).and_return(responses)
-      allow(responses).to receive(:create).and_return(response)
+      stub_completion(nil)
 
       expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error)
     end