Add authenticated endpoint and inference client for AI-powered Friendly Error Messages

.env.example

@@ -70,3 +70,8 @@ SCRATCH_ASSET_IMPORT_BASE_URL=https://example.com/assets/
 
 HOSTNAME=localhost
 LEARNER_EXPERIENCE_TESTS_DISPATCH_TOKEN=changeme
+
+# Friendly error messages
+FEM_INFERENCE_API_URL='http://localhost:11434/v1/'
+FEM_INFERENCE_API_KEY=''
+BYPASS_FEM_INFERENCE=true

.github/workflows/ci.yml

@@ -81,6 +81,7 @@ jobs:
       SALESFORCE_CONNECT_USER: postgres
       SALESFORCE_CONNECT_PASSWORD: password
       SALESFORCE_CONNECT_DB: salesforce_test
+      FEM_INFERENCE_API_URL: ''
     services:
       postgres:
         image: postgres:12

Gemfile

@@ -87,3 +87,5 @@ group :test do
   gem 'webdrivers'
   gem 'webmock'
 end
+
+gem 'openai', '~> 0.115.0'

Gemfile.lock

@@ -325,6 +325,9 @@ GEM
       stringio
       time
       uri
+    openai (0.115.0)
+      connection_pool (>= 2.2.3)
+      logger
     paper_trail (17.0.0)
       activerecord (>= 7.1)
       request_store (~> 1.4)
@@ -604,6 +607,7 @@ DEPENDENCIES
   omniauth-rails_csrf_protection (~> 2.0.1)
   omniauth-rpi!
   open-uri
+  openai (~> 0.115.0)
   paper_trail
   pg (~> 1.7)
   postmark-rails

app/controllers/api/features_controller.rb

@@ -9,7 +9,7 @@ def index
       school = current_user&.schools&.first
 
       enabled_feature_keys = Flipper.features
-                                    .select { |feature| Flipper.enabled?(feature.key, current_user, school) }
+                                    .select { |feature| FeatureFlagHelpers.enabled_for_user?(feature.key, current_user, school) }
                                     .map(&:key)
 
       render json: { enabled: enabled_feature_keys }

app/controllers/api/friendly_error_messages_controller.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Api
+  class FriendlyErrorMessagesController < ApiController
+    MAX_REQUEST_SIZE_KB = 100
+
+    before_action :authorize_user
+    skip_authorization_check only: :show
+
+    rescue_from FriendlyErrorMessagesInferenceClient::Error, with: :bad_gateway
+
+    def show
+      return render json: {}, status: :unauthorized unless FeatureFlagHelpers.enabled_for_user? :ai_friendly_errors, current_user, current_user&.schools&.first
+      return render json: { error: "The request was over the max size of #{MAX_REQUEST_SIZE_KB} KB" }, status: :content_too_large if request.body.size > MAX_REQUEST_SIZE_KB.kilobytes
+
+      params.require(%i[message trace runtime code])
+
+      fem = FriendlyErrorMessagesInferenceClient.new.get_message(
+        params[:message],
+        params[:trace],
+        params[:runtime],
+        params[:code]
+      )
+
+      render json: { fem: }, status: :ok
+    end
+  end
+end

app/controllers/api_controller.rb

@@ -60,6 +60,11 @@ def authorization_not_performed(exception)
     render json: { error: 'Internal server error' }, status: :internal_server_error
   end
 
+  def bad_gateway(exception)
+    Sentry.capture_exception(exception)
+    render_error_as_json exception, :bad_gateway
+  end
+
   def render_error_as_json(exception, status)
     render json: { error: "#{exception.class}: #{exception.message}" }, status:
   end

config/routes.rb

@@ -118,6 +118,8 @@
     post '/join/:join_code', to: 'join#create'
 
     resources :events, only: %i[create]
+
+    post '/friendly_error_messages', to: 'friendly_error_messages#show'
   end
 
   resource :github_webhooks, only: :create, defaults: { formats: :json }

lib/friendly_error_messages_inference_client.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+class FriendlyErrorMessagesInferenceClient
+  class Error < StandardError; end
+
+  MAX_RESULT_LENGTH_CHARS = 500
+
+  SYSTEM_PROMPT_BASE = <<~PROMPT
+    You help young people learn to program in Python.
+
+    A learner will show you some Python code, and usually an error message. Respond with a
+    short piece of feedback, addressed to them, that helps them find and fix the problem
+    themselves.
+  PROMPT
+
+  SYSTEM_PROMPT_L1_EXPLAIN = <<~PROMPT
+    The learner has only seen the raw Python error. Nothing has explained it to them yet.
+
+    Say in plain language what the error means, and suggest which part of their code to
+    look at. Pose it as a suggestion, not an instruction — "it might be worth checking…"
+    rather than "change…".
+
+    Three or four short sentences.
+
+    Do not write any code in this reply. No code blocks and no corrected lines.
+  PROMPT
+
+  def initialize
+    @client = OpenAI::Client.new(
+      base_url: ENV.fetch('FEM_INFERENCE_API_URL'),
+      api_key: ENV.fetch('FEM_INFERENCE_API_KEY', ''),
+      timeout: 15.0,
+      max_retries: 0
+    )
+  end
+
+  def get_message(message, trace, runtime, code)
+    return 'This is a placeholder friendly error message.' if ENV.fetch('BYPASS_FEM_INFERENCE', nil) == 'true'
+
+    result = @client.responses.create(
+      model: 'qwen3.5:9b',
+      input: [
+        { role: 'system', content: SYSTEM_PROMPT_BASE },
+        { role: 'system', content: SYSTEM_PROMPT_L1_EXPLAIN },
+        { role: 'user', content: JSON.dump({ message:, trace:, runtime:, code: }) }
+      ],
+      reasoning: { effort: :none }
+    ).output_text
+
+    raise Error, 'No message returned by inference service.' if result.empty?
+    raise Error, "The message by the inference service was too long - max #{MAX_RESULT_LENGTH_CHARS} chars, actual #{result.length} chars." if result.length > MAX_RESULT_LENGTH_CHARS
+
+    result
+  rescue OpenAI::Errors::APIError => e
+    raise Error, e.message
+  end
+end

lib/helpers/feature_flag_helpers.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class FeatureFlagHelpers
+  def self.enabled_for_user?(feature_key, user, school)
+    Flipper.enabled?(feature_key, user, school)
+  end
+end

spec/lib/friendly_error_messages_inference_client_spec.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe FriendlyErrorMessagesInferenceClient do
+  subject(:client) { described_class.new }
+
+  let(:mock_client) { instance_double(OpenAI::Client) }
+
+  context 'when BYPASS_FEM_INFERENCE is false' do
+    around do |example|
+      ClimateControl.modify(BYPASS_FEM_INFERENCE: 'false') do
+        example.run
+      end
+    end
+
+    it 'calls the inference server' do
+      responses = instance_double(OpenAI::Resources::Responses)
+      response = instance_double(OpenAI::Models::Responses::Response, output_text: 'A friendly error message')
+      allow(OpenAI::Client).to receive(:new).and_return(mock_client)
+      allow(mock_client).to receive(:responses).and_return(responses)
+      allow(responses).to receive(:create).and_return(response)
+
+      result = client.get_message('', '', '', '')
+      expect(result).to eq 'A friendly error message'
+    end
+
+    it 'throws an error when the inference call times out' do
+      responses = instance_double(OpenAI::Resources::Responses)
+      allow(OpenAI::Client).to receive(:new).and_return(mock_client)
+      allow(mock_client).to receive(:responses).and_return(responses)
+      allow(responses).to receive(:create).and_raise(OpenAI::Errors::APITimeoutError.new(url: 'http://some_url'))
+
+      expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error)
+    end
+
+    it 'throws an error when the inference server is down' do
+      responses = instance_double(OpenAI::Resources::Responses)
+      allow(OpenAI::Client).to receive(:new).and_return(mock_client)
+      allow(mock_client).to receive(:responses).and_return(responses)
+      allow(responses).to receive(:create).and_raise(OpenAI::Errors::APIConnectionError.new(url: 'http://some_url'))
+
+      expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error)
+    end
+
+    it 'throws an error when the inference server returns nothing' do
+      responses = instance_double(OpenAI::Resources::Responses)
+      response = instance_double(OpenAI::Models::Responses::Response, output_text: '')
+      allow(OpenAI::Client).to receive(:new).and_return(mock_client)
+      allow(mock_client).to receive(:responses).and_return(responses)
+      allow(responses).to receive(:create).and_return(response)
+
+      expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error)
+    end
+
+    it 'throws an error when the inference server returns a response that is too long' do
+      responses = instance_double(OpenAI::Resources::Responses)
+      response = instance_double(OpenAI::Models::Responses::Response, output_text: 'a' * 1000)
+      allow(OpenAI::Client).to receive(:new).and_return(mock_client)
+      allow(mock_client).to receive(:responses).and_return(responses)
+      allow(responses).to receive(:create).and_return(response)
+
+      expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error)
+    end
+  end
+
+  context 'when BYPASS_FEM_INFERENCE is true' do
+    around do |example|
+      ClimateControl.modify(BYPASS_FEM_INFERENCE: 'true') do
+        example.run
+      end
+    end
+
+    it 'returns a mock response' do
+      result = client.get_message('', '', '', '')
+      expect(result).to eq 'This is a placeholder friendly error message.'
+    end
+  end
+end

spec/requests/friendly_error_messages_spec.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe 'FriendlyErrorMessages' do
+  describe 'POST /friendly_error_messages' do
+    let(:headers) { { Authorization: UserProfileMock::TOKEN } }
+    let(:school) { create(:school) }
+    let(:student) { create(:student, school:) }
+    let(:mock_client) { instance_double(FriendlyErrorMessagesInferenceClient) }
+
+    before do
+      Flipper.enable(:ai_friendly_errors)
+    end
+
+    around do |example|
+      ClimateControl.modify(BYPASS_FEM_INFERENCE: 'true') do
+        example.run
+      end
+    end
+
+    it 'does not allow logged-out users' do
+      post '/api/friendly_error_messages'
+
+      expect(response).to have_http_status(:unauthorized)
+    end
+
+    it 'disallows requests over 100KB in size' do
+      authenticated_in_hydra_as(student)
+
+      post('/api/friendly_error_messages', headers:, params: { content: 'a' * 100.kilobytes })
+
+      expect(response).to have_http_status(:content_too_large)
+    end
+
+    it 'requires message, trace, runtime and code params' do
+      authenticated_in_hydra_as(student)
+
+      post('/api/friendly_error_messages', headers:, params: {
+             message: 'message',
+             trace: 'trace',
+             runtime: 'runtime'
+           })
+
+      expect(response).to have_http_status(:bad_request)
+    end
+
+    it 'is protected by a feature flag' do
+      Flipper.disable(:ai_friendly_errors)
+
+      authenticated_in_hydra_as(student)
+
+      post('/api/friendly_error_messages', headers:)
+
+      expect(response).to have_http_status(:unauthorized)
+    end
+
+    it 'returns a result for a well-formed request' do
+      authenticated_in_hydra_as(student)
+
+      post('/api/friendly_error_messages', headers:, params: {
+             message: 'message',
+             trace: 'trace',
+             runtime: 'runtime',
+             code: 'code'
+           })
+
+      expect(response).to have_http_status(:ok)
+      expect(response.parsed_body[:fem]).not_to be_nil
+    end
+
+    it 'returns a 503 when the client raises an error' do
+      authenticated_in_hydra_as(student)
+
+      allow(FriendlyErrorMessagesInferenceClient).to receive(:new).and_return(mock_client)
+      allow(mock_client).to receive(:get_message).and_raise(FriendlyErrorMessagesInferenceClient::Error)
+
+      post('/api/friendly_error_messages', headers:, params: {
+             message: 'message',
+             trace: 'trace',
+             runtime: 'runtime',
+             code: 'code'
+           })
+
+      expect(response).to have_http_status(:bad_gateway)
+    end
+  end
+end