feat: Add Twelve Data provider for exchange rates and securities (#2)

* feat: Add Twelve Data provider for exchange rates and securities

* test: fix hosting controller test, linting

* fix: add countries gem to handle country codes in Twelve Data provider

* fix: allow security search combobox to have no logo

* refactor: update Twelve Data provider use time series endpoint

* fix: set twelve data as default provider

.env.example

@@ -22,6 +22,15 @@ SECRET_KEY_BASE=secret-value
 # Get it here: https://synthfinance.com/
 SYNTH_API_KEY=
 
+# Optional: Twelve Data API Key for exchange rates + stock prices
+# (you can also set this in your self-hosted settings page)
+# Get it here: https://twelvedata.com/
+TWELVE_DATA_API_KEY=
+
+# Optional: Twelve Data provider is the default for exchange rates and securities.
+EXCHANGE_RATE_PROVIDER=twelve_data
+SECURITIES_PROVIDER=twelve_data
+
 # Custom port config
 # For users who have other applications listening at 3000, this allows them to set a value puma will listen to.
 PORT=3000

.env.local.example

@@ -3,3 +3,4 @@ SELF_HOSTED=false
 
 # Enable Synth market data (careful, this will use your API credits)
 SYNTH_API_KEY=yourapikeyhere
+TWELVE_DATA_API_KEY=yourapikeyhere

Gemfile

@@ -52,6 +52,7 @@ gem "ostruct"
 gem "bcrypt", "~> 3.1"
 gem "jwt"
 gem "jbuilder"
+gem "countries"
 
 # OAuth & API Security
 gem "doorkeeper"

Gemfile.lock

@@ -140,6 +140,8 @@ GEM
     climate_control (1.2.0)
     concurrent-ruby (1.3.5)
     connection_pool (2.5.3)
+    countries (8.0.3)
+      unaccent (~> 0.3)
     crack (1.0.0)
       bigdecimal
       rexml
@@ -576,6 +578,7 @@ GEM
       railties (>= 7.1.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
+    unaccent (0.4.0)
     unicode (0.4.4.5)
     unicode-display_width (3.1.4)
       unicode-emoji (~> 4.0, >= 4.0.4)
@@ -629,6 +632,7 @@ DEPENDENCIES
   brakeman
   capybara
   climate_control
+  countries
   csv
   debug
   derailed_benchmarks

app/controllers/settings/hostings_controller.rb

@@ -8,6 +8,9 @@ class Settings::HostingsController < ApplicationController
   def show
     synth_provider = Provider::Registry.get_provider(:synth)
     @synth_usage = synth_provider&.usage
+
+    twelve_data_provider = Provider::Registry.get_provider(:twelve_data)
+    @twelve_data_usage = twelve_data_provider&.usage
   end
 
   def update
@@ -23,6 +26,10 @@ class Settings::HostingsController < ApplicationController
       Setting.synth_api_key = hosting_params[:synth_api_key]
     end
 
+    if hosting_params.key?(:twelve_data_api_key)
+      Setting.twelve_data_api_key = hosting_params[:twelve_data_api_key]
+    end
+
     redirect_to settings_hosting_path, notice: t(".success")
   rescue ActiveRecord::RecordInvalid => error
     flash.now[:alert] = t(".failure")
@@ -36,7 +43,7 @@ class Settings::HostingsController < ApplicationController
 
   private
     def hosting_params
-      params.require(:setting).permit(:require_invite_for_signup, :require_email_confirmation, :synth_api_key)
+      params.require(:setting).permit(:require_invite_for_signup, :require_email_confirmation, :synth_api_key, :twelve_data_api_key)
     end
 
     def ensure_admin

app/models/exchange_rate/provided.rb

@@ -3,8 +3,9 @@ module ExchangeRate::Provided
 
   class_methods do
     def provider
+      provider = ENV["EXCHANGE_RATE_PROVIDER"] || "twelve_data"
       registry = Provider::Registry.for_concept(:exchange_rates)
-      registry.get_provider(:synth)
+      registry.get_provider(provider.to_sym)
     end
 
     def find_or_fetch_rate(from:, to:, date: Date.current, cache: true)

app/models/provider/registry.rb

@@ -40,6 +40,14 @@ class Provider::Registry
         Provider::Synth.new(api_key)
       end
 
+      def twelve_data
+        api_key = ENV.fetch("TWELVE_DATA_API_KEY", Setting.twelve_data_api_key)
+
+        return nil unless api_key.present?
+
+        Provider::TwelveData.new(api_key)
+      end
+
       def plaid_us
         config = Rails.application.config.plaid
 
@@ -92,9 +100,9 @@ class Provider::Registry
     def available_providers
       case concept
       when :exchange_rates
-        %i[synth]
+        %i[synth twelve_data]
       when :securities
-        %i[synth]
+        %i[synth twelve_data]
       when :llm
         %i[openai]
       else

app/models/provider/twelve_data.rb

@@ -0,0 +1,195 @@
+class Provider::TwelveData < Provider
+  include ExchangeRateConcept, SecurityConcept
+
+  # Subclass so errors caught in this provider are raised as Provider::TwelveData::Error
+  Error = Class.new(Provider::Error)
+  InvalidExchangeRateError = Class.new(Error)
+  InvalidSecurityPriceError = Class.new(Error)
+
+  def initialize(api_key)
+    @api_key = api_key
+  end
+
+  def healthy?
+    with_provider_response do
+      response = client.get("#{base_url}/api_usage")
+      JSON.parse(response.body).dig("plan_category").present?
+    end
+  end
+
+  def usage
+    with_provider_response do
+      response = client.get("#{base_url}/api_usage")
+
+      parsed = JSON.parse(response.body)
+
+      limit = parsed.dig("plan_daily_limit")
+      used = parsed.dig("daily_usage")
+      remaining = limit - used
+
+      UsageData.new(
+        used: used,
+        limit: limit,
+        utilization: used / limit * 100,
+        plan: parsed.dig("plan_category"),
+      )
+    end
+  end
+
+  # ================================
+  #          Exchange Rates
+  # ================================
+
+  def fetch_exchange_rate(from:, to:, date:)
+    with_provider_response do
+      response = client.get("#{base_url}/exchange_rate") do |req|
+        req.params["symbol"] = "#{from}/#{to}"
+        req.params["date"] = date.to_s
+      end
+
+      rate = JSON.parse(response.body).dig("rate")
+
+      Rate.new(date: date.to_date, from:, to:, rate: rate)
+    end
+  end
+
+  def fetch_exchange_rates(from:, to:, start_date:, end_date:)
+    with_provider_response do
+      response = client.get("#{base_url}/time_series") do |req|
+        req.params["symbol"] = "#{from}/#{to}"
+        req.params["start_date"] = start_date.to_s
+        req.params["end_date"] = end_date.to_s
+        req.params["interval"] = "1day"
+      end
+
+      data = JSON.parse(response.body).dig("values")
+      data.map do |resp|
+        rate = resp.dig("close")
+        date = resp.dig("datetime")
+        if rate.nil?
+          Rails.logger.warn("#{self.class.name} returned invalid rate data for pair from: #{from} to: #{to} on: #{date}.  Rate data: #{rate.inspect}")
+          next
+        end
+
+        Rate.new(date: date.to_date, from:, to:, rate:)
+      end.compact
+    end
+  end
+
+  # ================================
+  #           Securities
+  # ================================
+
+  def search_securities(symbol, country_code: nil, exchange_operating_mic: nil)
+    with_provider_response do
+      response = client.get("#{base_url}/symbol_search") do |req|
+        req.params["symbol"] = symbol
+        req.params["outputsize"] = 25
+      end
+
+      parsed = JSON.parse(response.body)
+
+      parsed.dig("data").map do |security|
+        country = ISO3166::Country.find_country_by_any_name(security.dig("country"))
+
+        Security.new(
+          symbol: security.dig("symbol"),
+          name: security.dig("instrument_name"),
+          logo_url: nil,
+          exchange_operating_mic: security.dig("mic_code"),
+          country_code: country ? country.alpha2 : nil
+        )
+      end
+    end
+  end
+
+  def fetch_security_info(symbol:, exchange_operating_mic:)
+    with_provider_response do
+      response = client.get("#{base_url}/profile") do |req|
+        req.params["symbol"] = symbol
+        req.params["mic_code"] = exchange_operating_mic
+      end
+
+      profile = JSON.parse(response.body)
+
+      response = client.get("#{base_url}/logo") do |req|
+        req.params["symbol"] = symbol
+        req.params["mic_code"] = exchange_operating_mic
+      end
+
+      logo = JSON.parse(response.body)
+
+      SecurityInfo.new(
+        symbol: symbol,
+        name: profile.dig("name"),
+        links: profile.dig("website"),
+        logo_url: logo.dig("url"),
+        description: profile.dig("description"),
+        kind: profile.dig("type"),
+        exchange_operating_mic: exchange_operating_mic
+      )
+    end
+  end
+
+  def fetch_security_price(symbol:, exchange_operating_mic: nil, date:)
+    with_provider_response do
+      historical_data = fetch_security_prices(symbol:, exchange_operating_mic:, start_date: date, end_date: date)
+
+      raise ProviderError, "No prices found for security #{symbol} on date #{date}" if historical_data.data.empty?
+
+      historical_data.data.first
+    end
+  end
+
+  def fetch_security_prices(symbol:, exchange_operating_mic: nil, start_date:, end_date:)
+    with_provider_response do
+      response = client.get("#{base_url}/time_series") do |req|
+        req.params["symbol"] = symbol
+        req.params["mic_code"] = exchange_operating_mic
+        req.params["start_date"] = start_date.to_s
+        req.params["end_date"] = end_date.to_s
+        req.params["interval"] = "1day"
+      end
+
+      parsed = JSON.parse(response.body)
+      parsed.dig("values").map do |resp|
+        price = resp.dig("close")
+        date = resp.dig("datetime")
+        if price.nil?
+          Rails.logger.warn("#{self.class.name} returned invalid price data for security #{symbol} on: #{date}.  Price data: #{price.inspect}")
+          next
+        end
+
+        Price.new(
+          symbol: symbol,
+          date: date.to_date,
+          price: price,
+          currency: parsed.dig("currency"),
+          exchange_operating_mic: exchange_operating_mic
+        )
+      end.compact
+    end
+  end
+
+  private
+    attr_reader :api_key
+
+    def base_url
+      ENV["TWELVE_DATA_URL"] || "https://api.twelvedata.com"
+    end
+
+    def client
+      @client ||= Faraday.new(url: base_url) do |faraday|
+        faraday.request(:retry, {
+          max: 2,
+          interval: 0.05,
+          interval_randomness: 0.5,
+          backoff_factor: 2
+        })
+
+        faraday.request :json
+        faraday.response :raise_error
+        faraday.headers["Authorization"] = "apikey #{api_key}"
+      end
+    end
+end

app/models/security/provided.rb

@@ -5,8 +5,9 @@ module Security::Provided
 
   class_methods do
     def provider
+      provider = ENV["SECURITIES_PROVIDER"] || "twelve_data"
       registry = Provider::Registry.for_concept(:securities)
-      registry.get_provider(:synth)
+      registry.get_provider(provider.to_sym)
     end
 
     def search_provider(symbol, country_code: nil, exchange_operating_mic: nil)

app/models/setting.rb

@@ -3,6 +3,7 @@ class Setting < RailsSettings::Base
   cache_prefix { "v1" }
 
   field :synth_api_key, type: :string, default: ENV["SYNTH_API_KEY"]
+  field :twelve_data_api_key, type: :string, default: ENV["TWELVE_DATA_API_KEY"]
   field :openai_access_token, type: :string, default: ENV["OPENAI_ACCESS_TOKEN"]
 
   field :require_invite_for_signup, type: :boolean, default: false

app/views/securities/_combobox_security.turbo_stream.erb

@@ -1,5 +1,7 @@
 <div class="flex items-center">
-  <%= image_tag(combobox_security.logo_url, class: "rounded-full h-8 w-8 inline-block mr-2" ) %>
+    <% if combobox_security.logo_url.present? %>
+      <%= image_tag(combobox_security.logo_url, class: "rounded-full h-8 w-8 inline-block mr-2" ) %>
+    <% end %>
   <div class="flex items-center justify-between w-full">
     <div class="flex flex-col">
       <span class="text-sm font-medium">

app/views/settings/_section.html.erb

@@ -6,7 +6,7 @@
       <p class="text-secondary text-sm mt-1"><%= subtitle %></p>
     <% end %>
   </div>
-  <div>
+  <div class="space-y-4">
     <%= content %>
   </div>
 </section>

app/views/settings/hostings/_twelve_data_settings.html.erb

@@ -0,0 +1,49 @@
+<div class="space-y-4">
+  <div>
+    <h2 class="font-medium mb-1"><%= t(".title") %></h2>
+    <% if ENV["TWELVE_DATA_API_KEY"].present? %>
+      <p class="text-sm text-secondary">You have successfully configured your Twelve Data API key through the TWELVE_DATA_API_KEY environment variable.</p>
+    <% else %>
+      <p class="text-secondary text-sm mb-4"><%= t(".description") %></p>
+    <% end %>
+  </div>
+
+  <%= styled_form_with model: Setting.new,
+                       url: settings_hosting_path,
+                       method: :patch,
+                       data: {
+                         controller: "auto-submit-form",
+                         "auto-submit-form-trigger-event-value": "blur"
+                       } do |form| %>
+    <%= form.text_field :twelve_data_api_key,
+                        label: t(".label"),
+                        type: "password",
+                        placeholder: t(".placeholder"),
+                        value: ENV.fetch("TWELVE_DATA_API_KEY", Setting.twelve_data_api_key),
+                        disabled: ENV["TWELVE_DATA_API_KEY"].present?,
+                        container_class: @twelve_data_usage.present? && !@twelve_data_usage.success? ? "border-red-500" : "",
+                        data: { "auto-submit-form-target": "auto" } %>
+  <% end %>
+
+  <% if @twelve_data_usage.present? && @twelve_data_usage.success? %>
+    <div class="space-y-4">
+      <div class="space-y-2">
+        <p class="text-sm text-secondary">
+          <%= t(".api_calls_used",
+                used: number_with_delimiter(@twelve_data_usage.data.used),
+                limit: number_with_delimiter(@twelve_data_usage.data.limit),
+                percentage: number_to_percentage(@twelve_data_usage.data.utilization, precision: 1)) %>
+        </p>
+        <div class="w-52 h-1.5 bg-gray-100 rounded-2xl">
+          <div class="h-full bg-green-500 rounded-2xl"
+               style="width: <%= [@twelve_data_usage.data.utilization, 2].max %>%;"></div>
+        </div>
+      </div>
+      <div class="bg-gray-100 rounded-md px-1.5 py-0.5 w-fit">
+        <p class="text-xs font-medium text-secondary uppercase">
+          <%= t(".plan", plan: @twelve_data_usage.data.plan) %>
+        </p>
+      </div>
+    </div>
+  <% end %>
+</div>

app/views/settings/hostings/show.html.erb

@@ -4,6 +4,9 @@
   <div class="space-y-6">
     <%= render "settings/hostings/synth_settings" %>
   </div>
+  <div class="space-y-6">
+    <%= render "settings/hostings/twelve_data_settings" %>
+  </div>
 <% end %>
 
 <%= settings_section title: t(".invites") do %>

config/locales/views/settings/hostings/en.yml

@@ -28,6 +28,13 @@ en:
         placeholder: Enter your API key here
         plan: "%{plan} plan"
         title: Synth Settings
+      twelve_data_settings:
+        api_calls_used: "%{used} / %{limit} API daily calls used (%{percentage})"
+        description: Input the API key provided by Twelve Data
+        label: API Key
+        placeholder: Enter your API key here
+        plan: "%{plan} plan"
+        title: Twelve Data Settings
       update:
         failure: Invalid setting value
         success: Settings updated

test/controllers/settings/hostings_controller_test.rb

@@ -9,6 +9,7 @@ class Settings::HostingsControllerTest < ActionDispatch::IntegrationTest
 
     @provider = mock
     Provider::Registry.stubs(:get_provider).with(:synth).returns(@provider)
+    Provider::Registry.stubs(:get_provider).with(:twelve_data).returns(@provider)
     @usage_response = provider_success_response(
       OpenStruct.new(
         used: 10,
@@ -31,6 +32,7 @@ class Settings::HostingsControllerTest < ActionDispatch::IntegrationTest
 
   test "should get edit when self hosting is enabled" do
     @provider.expects(:usage).returns(@usage_response)
+    @provider.expects(:usage).returns(@usage_response)
 
     with_self_hosting do
       get settings_hosting_url

test/models/account/market_data_importer_test.rb

@@ -18,7 +18,7 @@ class Account::MarketDataImporterTest < ActiveSupport::TestCase
     @provider = mock("provider")
     Provider::Registry.any_instance
                       .stubs(:get_provider)
-                      .with(:synth)
+                      .with(:twelve_data)
                       .returns(@provider)
   end
 

test/models/market_data_importer_test.rb

@@ -17,7 +17,7 @@ class MarketDataImporterTest < ActiveSupport::TestCase
     @provider = mock("provider")
     Provider::Registry.any_instance
                       .stubs(:get_provider)
-                      .with(:synth)
+                      .with(:twelve_data)
                       .returns(@provider)
   end
 

test/system/settings_test.rb

@@ -34,6 +34,7 @@ class SettingsTest < ApplicationSystemTestCase
   test "can update self hosting settings" do
     Rails.application.config.app_mode.stubs(:self_hosted?).returns(true)
     Provider::Registry.stubs(:get_provider).with(:synth).returns(nil)
+    Provider::Registry.stubs(:get_provider).with(:twelve_data).returns(nil)
     open_settings_from_sidebar
     assert_selector "li", text: "Self hosting"
     click_link "Self hosting"