Computer-aided Day Trading

tids  = Ticker.where(:exchange => 'NYSE').all.map {|t| t.id }
debut = Time.parse('1 jan 2019')
fin   = Time.parse('31 dec 2019')
bars  = Bar.where(:date => debut..fin, :ticker_id => tids)
           .order(:ticker_id, Sequel.asc(:date))
           .all
groups  = bars.group_by {|b| b.ticker_id }
changes = groups.map do |ticker_id, bars|
  bars.each_cons(2).map {|a, b| b.change_from a }
end.flatten

# This will sort the changes into the bucket that are closest in value
hist = changes.histogram [-0.35, -0.25, -0.15, -0.05, 0.05, 0.15, 0.25, 0.35]

Redefined in section 4

2019
"Fractional 2-day Change" => "# of Occurrences"
               -100..-0.3 => 99
               -0.3..-0.2 => 243
               -0.2..-0.1 => 2532
               -0.1.. 0   => 267752
                0  .. 0.1 => 309317
                0.1.. 0.2 => 3218
                0.2.. 0.3 => 377
                0.3.. 100 => 151

Redefined in section 4

2015
"Fractional 2-day Change" => "# of Occurrences"
               -100..-0.3 => 36
               -0.3..-0.2 => 192
               -0.2..-0.1 => 2171
               -0.1.. 0   => 237530
                0  .. 0.1 => 243177
                0.1.. 0.2 => 2490
                0.2.. 0.3 => 238
                0.3.. 100 => 78

Redefined in section 4

      assessor.buy_when :history => 2 do |history|
        today     = history[-1]
        yesterday = history[-2]
      
        [[today.change_from(yesterday) <= drop,
          today.change_from(today)     <= drop].any?,
  
         today.rank <= rank,
         today.close >= min
        ].all?
      end

# {"SYM" => [Rank, Value]}
rankings = Ticker.rankings NYSE, :date => date
rankings[ticker.symbol] #=> [Rank, Value]

0.00.step(:to => 0.1, :by => 0.005) do |m|
  m = -m
  0.step(:to => 6, :by => 0.1) do |b|
    sim.m = m
    sim.b = b
    sells = sim.assess_sells

    # ...
  end
end

      # m = -0.02, b = 5.2
      assessor.sell_when do |original, today|
        days_held = today.trading_days_from original
        
        sell_point = [@m * days_held + @b, 0].max
      
        today.change_from(original) >= sell_point
      end

"In the cumulative timespan of 2018-2020"

[drop = 20%, m = -0.02, b = 5.2] => ROI: 0.9839851521267415,
                                    Profits: 21.912217326647706,
                                    Hold: 216 days,
                                    Pieces: 50

Yearly Breakdown

| Timeframe |    m  |  b  | Buys | Median Hold | Skips |  Mean ROI  | Reinvested Profits|
|-----------|-------|-----|------|-------------|-------|------------|------------|
|   2018    | -0.02 | 5.2 |   36 |        619  |   0   | -0.14905347|   $8.93 |
|   2019    | -0.02 | 5.2 |   56 |        258  |   4   | 0.322075814|  $14.18 |
|   2020    | -0.02 | 5.2 |  326 |        202  | 235   | 1.222808200|  $28.90 |

module Algorithms
  class VolatileDrop < Simulator
    attr_accessor :m
    attr_accessor :b
    attr_accessor :drop
    attr_accessor :rise # TODO get this out of here

    FOLDER = "volatile_drop"

    # use 23 pieces
    def initialize(stocks:  nil,
                   after:   nil,
                   before:  nil,
                   drop:   -0.2,
                   rank:    60,
                   m:      -0.02,
                   b:       5.2,
                   min:     0.4,
                   **extra)
      super(:stocks => stocks,
            :after  => after,
            :before => before)
      @drop = drop
      @m = m
      @b = b
  
      assessor.buy_when :history => 2 do |history|
        today     = history[-1]
        yesterday = history[-2]
      
        [[today.change_from(yesterday) <= drop,
          today.change_from(today)     <= drop].any?,
  
         today.rank <= rank,
         today.close >= min
        ].all?
      end
      
      # for ROI: m = -0.03, b = 3.0
      # for $$$: m = -0.02, b = 5.2
      #      or: m = -0.00, b = 0.6
      #      (those two average roughly the same from 2008-2020)
      assessor.sell_when do |original, today|
        days_held = today.trading_days_from original
        
        sell_point = [@m * days_held + @b, 0].max
      
        today.change_from(original) >= sell_point
      end
    end
  end
end

require './assessor.rb'

class Simulator
  attr_accessor :assessor
  attr_accessor :stocks
  attr_accessor :after
  attr_accessor :before

  def initialize(stocks:  nil,
                 after:   nil,
                 before:  nil)
    @stocks = stocks
    @after  = after
    @before = before
    @assessor = Assessor.new
  end

  # TODO rebuild all caches in the commented-out format
  def cache_name
    vars = instance_variables - [:@after, :@before, :@stocks, :@assessor]

    #"data/#{self.class::FOLDER}/" +
    #"#{[after.year.to_s, before.year.to_s].uniq.join "-"}_"+ 
    #"#{after.to_i}-#{before.to_i}_" +
    #vars.sort.map {|v| v.to_s[1] + instance_variable_get(v).to_s }.join("_") +
    #".sim"

    "data/#{self.class::FOLDER}/" +
    "#{[after.year.to_s, before.year.to_s].uniq.join "-"}_" +
    vars.map {|v| v.to_s[1] + instance_variable_get(v).to_s }.join("_") +
    ".sim"
  end

  def assess_buys
    @assessor.assess_buys @stocks, :after  => @after,
                                   :before => @before,
                                   :force  => @force
  end

  def assess_sells(partial: false)
    @assessor.assess_sells :partial => partial
  end

  def run
    assess_buys
    assess_sells
  end

  def results
    @assessor.results
  end

  def results=(val)
    @assessor.results = val
  end

  def holding
    @assessor.holding
  end

  def holding=(val)
    @assessor.holding = val
  end

  # Maybe `h[:hold]` should always be filled out?
  # Same with `h[:latest]`?
  def still_negative
    unsold = results.filter {|h| h[:sold] == nil }

    ticks = unsold.map {|h| h[:buy].ticker }
    #latests = Bar.where(:ticker => ticks)
    #             .order(Sequel.desc(:date))
    #             .group(:ticker_id)
    #             .all
    latests = ticks.map do |t|
      [t, Bar.where(:ticker => t)
             .order(Sequel.desc(:date))
             .first]
    end.to_h

    unsold.each do |h|
      h[:latest] = latests[h[:buy].ticker]
      h[:ROI]    = h[:latest].change_from h[:buy]
      h[:hold]   = h[:latest].trading_days_from h[:buy]
    end

    unsold.filter {|h| h[:ROI] < 0 }
  end

  def stats
    statz  = {:date        => after..before,
              :buys        => results.size,
              :unsold      => results.filter {|h| h[:sell].nil? }.size,
              :delisted    => results.filter {|h| h[:delisted] }.size,
              :median_hold => results.map {|h| h[:hold] || 1000 }.median,
              :sp500       => spy(after, before),
              :mean_ROI    => results.map {|h| h[:ROI] }.mean,
              :median_ROI  => results.map {|h| h[:ROI] }.median,
              :stddev_ROI  => results.map {|h| h[:ROI] }.standard_deviation
             }
    statz[:sharpe] = statz[:mean_ROI] / statz[:stddev_ROI]

    statz
  end
end

Dir['./algos/*.rb'].each {|f| require f }

Algorithm = eval("Algorithms::#{CONFIG[:algorithm]}")

class Assessor
  attr_accessor :buying_plan
  attr_accessor :selling_plan
  attr_accessor :history_requirement

  attr_accessor :holding
  attr_accessor :results

  DELISTING_DEADBAND = 14.days

  def buy_when(history: 2, &b)
    @buying_plan = b
    @history_requirement = history
  end

  def sell_when(&b)
    @selling_plan = b
  end

  def buy?(ticker)
    buying_plan[ticker]
  end

  def sell?(ticker, original)
    selling_plan[ticker, original]
  end

  def assess_buys(tickers, opts={})
    tids = tickers.map {|t| t.id }

    debut  = Time.parse(opts[:after].to_s  || '1 march 1900')
    fin    = Time.parse(opts[:before].to_s || Date.today.to_s)

    bars   = Bar.where(:date => debut..fin, :ticker_id => tids)
                .order(:ticker_id, Sequel.asc(:date))
                .all
    groups = bars.group_by {|b| b.ticker_id }

    @holding = []

    # create groups of size `@history_requirement`, and then
    # pass that history to the checker
    # most recent bar is at -1, oldest bar is at 0
    @holding = groups.map do |ticker_id, bars|
      # assume the history is 
      histories = bars.each_cons history_requirement

      histories.filter do |history|

        # verify that the history is consecutive
        day_deltas = history.each_cons(2).map {|a, b| b.date - a.date }

        if day_deltas.any? {|v| v > 4.days }
          false
        else
          buy? history
        end
      end.map {|history| history.last }
    end.flatten

    # `@holding` currently references the days that a decision to buy is made
    # (using the day's closing price), but we don't *actually* buy until the
    # next morning. So we need to replace these stocks with the next day's
    # stock.
    # 
    # This is key because the `Bar#change_from` method operates on the opening
    # price of the earlier day.
    #
    # If `bars[index + 1]` is nil because we're dealing with some HOT OF THE
    # PRESS stock recommendations, then... I don't really have a plan for that
    # yet.  Then the stock doesn't exist, so just present the stock itself.
    # It'll stay until the time period is recalculated, which happens often.
    #
    # From here on out, we're dealing with *simulation*.
    @holding = @holding.map do |stock|
      bars  = Bar.where(:ticker => stock.ticker,
                        :date => stock.date..(stock.date + 7.days))
                 .order(Sequel.asc(:date))
                 .all
      index = bars.index stock
      bars[index + 1] || stock
    end
  end

  def assess_sells(partial: false)
    # assumes `@holding` and `@results` are accurately mapped
    if partial
      verified = @results.filter {|h| h[:sell] }
      unverified_stocks = @results.filter {|h| h[:sell].nil? }
                                  .map    {|h| h[:buy] }
    else
      unverified_stocks, verified = @holding, []
    end

    # Stocks can be delisted, at which point stocks held will be no longer
    # valid, but then a *new* ticker can start and can *reuse* the old name.
    # And since any stocks held from the previous incarnation won't be valid
    # for the new incarnation of the symbol, we need to separate those
    # instances. We do this by looking for a stretch of 7 days (using the date,
    # not the trading days, since trading days is calculated based on the
    # availability of bar information for that specific stock) during which the
    # stock is not traded (stocks can go intermittently inactive for short
    # periods of time, but that doesn't imply delistment).
    unverified = unverified_stocks.map do |stock|
      bars     = Bar.where(:ticker => stock.ticker) { date >= stock.date }
                    .order(Sequel.asc(:date))
                    .all
      periods = bars.slice_when do |before, after|
        after.date - before.date >= DELISTING_DEADBAND
      end

      # this is the only period that starts from the holding bar
      range = periods.first

      sell_bar = range.find {|day| sell? stock, day }
      delisted = if sell_bar
                   false
                 else 
                   Time.now - range.last.date >= DELISTING_DEADBAND
                 end

      {:buy  => stock,
       :sell => sell_bar,
       :hold => sell_bar ? sell_bar.trading_days_from(stock)  : nil,
       :ROI  => sell_bar ? sell_bar.change_from(stock) : -1,
       :delisted => delisted
      }
    end

    @results = (unverified + verified).sort_by {|h| h[:buy].date }
  end

  def assess(tickers, opts={})
    assess_buys tickers, opts
    assess_sells :partial => opts[:partial]
  end
end

require 'yaml'
Dir.chdir File.dirname(File.expand_path(__FILE__))
CONFIG = YAML.load File.read("config.yml")

require 'open-uri'
require 'alpaca/trade/api'
require 'alphavantagerb'
require './db.rb'
require './simulator.rb'
require 'kder'
require 'histogram/array'

{Configure the Market APIs, 11}

Alpaca::Trade::Api.configure do |config|
  config.endpoint   = "https://api.alpaca.markets"
  config.key_id     = CONFIG[:Alpaca][:ID]
  config.key_secret = CONFIG[:Alpaca][:secret]
end

ALP_CLIENT = Alpaca::Trade::Api::Client.new
AV_CLIENT  = Alphavantage::Client.new :key => CONFIG[:AV][:key]

{Enhance the Alpaca Ruby API, 12}

Used in section 10

class Alpaca::Trade::Api::Client
  # This takes care of the issue where I was not able to provide other options
  # to the GET request. Now, I can specify "before" and "after" IAW the API.
  def bars(timeframe, symbols, opts={})
    opts[:limit] ||= 100
    opts[:symbols] = symbols.join(',')

    validate_timeframe(timeframe)
    response = get_request(data_endpoint, "v1/bars/#{timeframe}", opts)
    json = JSON.parse(response.body)
    json.keys.each_with_object({}) do |symbol, hash|
      hash[symbol] = json[symbol].map { |bar| Alpaca::Trade::Api::Bar.new(bar) }
    end
  end 

  # Enabling me to use the "qty" query parameter. Playing it extra safe
  # by not even sending the parameter unless there's a specified number
  def close_position(symbol: nil, qty: nil)
    response = delete_request(endpoint,
                              "v2/positions/#{symbol}#{qty ? "?qty=#{qty}" : ""}")
    raise NoPositionForSymbol,
          JSON.parse(response.body)['message'] if response.status == 404

    Position.new(JSON.parse(response.body))
  end
end

{Market Abstraction, 12}

Used in section 11

module Market
  module Stock
    extend self

    CLOSE = "16:00" # closing time of the markets
    DELAY = 15 * 60 # how long to wait (in sec) before grabbing data

    # Alpaca download but don't install
    def download(tickers, opts={})
      span = opts.delete(:span) || 'day'
      opts[:limit] ||= 1000

      opts.each do |k, v| 
        if [String, Date, DateTime, Time].include? v.class
          opts[k] = DateTime.parse(v.to_s).to_s
        end
      end

      # `CLIENT.bars` returns a hash, so this will also merge them all
      # into one. key collision will only happen if the key is duplicated
      # in the `ticker` argument.
      symbols = tickers.map {|t| t.symbol }
      data    = symbols.each_slice(50).map do |ticks|
        ALP_CLIENT.bars span, ticks, opts
      end.inject({}) {|h, v| h.merge v }

      # strip out any bar that could be from today's incomplete data
      data.each do |sym, bars|
        bars.delete_if do |bar|
          bar.date == Time.parse(Date.today.to_s) &&
          Time.now < (Time.parse(CLOSE) + DELAY)
        end
      end

      # provide a hash so that we can get the ID without
      # fetching the symbol from the DB
      tids = tickers.map {|t| [t.symbol, t] }.to_h

      # Put the data in a hash format so that it's consistent with the
      # AlphaVantage style and allows for easy use with DB#multi_insert
      data.map do |sym, bars|
        [sym, bars.map do |bar|
          {:date   => bar.time,
           :open   => bar.open,
           :high   => bar.high,
           :low    => bar.low,
           :close  => bar.close,
           :volume => bar.volume,
           :span   => 'day',
           :ticker_id => tids[sym].id
          }
        end]
      end.to_h
    end

    def install(tickers, opts={})
      return {} if opts[:after] == Time.parse(Date.today.to_s)
      return {} if opts[:after] == Time.parse(Date.today.to_s) - 1.day &&
                   Time.now < (Time.parse(CLOSE) + DELAY)

      updates = download tickers, opts
      DB[:bars].multi_insert updates.values.flatten

      updates
    end

    # can only do one stock at a time
    # AlphaVantage
    def download_stock(ticker, after: '1900-01-01', before: Date.today.strftime("%Y-%m-%d"))
      stock  = AV_CLIENT.stock :symbol => ticker.symbol
      series = stock.timeseries :outputsize => 'full'

      bars = series.output['Time Series (Daily)']
      bars = bars.filter {|k, bar| k > after && k < before }

      bars.map do |k, bar|
        {:date   => Time.parse(k),
         :open   => bar['1. open'].to_f,
         :high   => bar['2. high'].to_f,
         :low    => bar['3. low'].to_f,
         :close  => bar['4. close'].to_f,
         :volume => bar['5. volume'].to_i,
         :span   => 'day',
         :ticker_id => ticker.id
        }
      end
    end

    def install_stock(stock, **kwargs)
      data = download_stock stock, **kwargs
      DB[:bars].multi_insert data
    end
  end

  module Futures
    def download(future: nil, after: '1900-01-01', before: Date.today.strftime("%Y-%m-%d"))
      url = "https://query1.finance.yahoo.com/v7/finance/download/" +
            "#{future.ymbol}?" +
            "period1=#{after.to_i}&" +
            "period2=#{before.to_i}&" +
            "interval=1d&events=history&includeAdjustedClose=true"
      user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) " +
                   "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 " +
                   "Safari/605.1.15"
      data = URI.open(url, "User-Agent" => user_agent) do |site|
        site.read
      end

      data.split("\n").map {|line| line.split "," }.map do |line|
        {:date  => Time.parse(line[0]),
         :open  => line[1].to_f,
         :high  => line[2].to_f,
         :low   => line[3].to_f,
         :close => line[5].to_f,
         :volume => line[6].to_f}
      end
    end
  end
end