Skip to content

Show all combinations prompt selection algorithm #43

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lukebaker wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Show all combinations prompt selection algorithm #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c7684e9
Add all-combos algorithm
lukebaker Jun 15, 2018
44eb9c7
Add tests to verify functionality of all-combos
lukebaker Jun 18, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 104 additions & 57 deletions app/models/question.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
class Question < ActiveRecord::Base
acts_as_versioned

require 'set'
include Utility
extend ActiveSupport::Memoizable

belongs_to :creator, :class_name => "Visitor", :foreign_key => "creator_id"
belongs_to :site, :class_name => "User", :foreign_key => "site_id"

has_many :choices, :order => 'score DESC'
has_many :prompts do
def pick(algorithm = nil)
Expand Down Expand Up @@ -54,7 +54,7 @@ def create_choices_from_ideas
end
end
end

def item_count
choices.size
end
Expand All @@ -70,29 +70,76 @@ def median_votes_per_session
end

def choose_prompt(options = {})

# if there is one or fewer active choices, we won't be able to find a prompt
if self.choices.size - self.inactive_choices_count <= 1
# if there is one or fewer active choices, we won't be able to find a prompt
if self.choices.size - self.inactive_choices_count <= 1
raise RuntimeError, "More than one choice needs to be active"
end

if self.uses_catchup? || options[:algorithm] == "catchup"
logger.info("Question #{self.id} is using catchup algorithm!")
next_prompt = self.pop_prompt_queue
if next_prompt.nil?
logger.info("DEBUG Catchup prompt cache miss! Nothing in prompt_queue")
next_prompt = self.simple_random_choose_prompt
record_prompt_cache_miss
if options[:algorithm] == "all-combos" && options[:visitor]
self.all_combos_prompt_selection(options[:visitor])
elsif self.uses_catchup? || options[:algorithm] == "catchup"
logger.info("Question #{self.id} is using catchup algorithm!")
next_prompt = self.pop_prompt_queue
if next_prompt.nil?
logger.info("DEBUG Catchup prompt cache miss! Nothing in prompt_queue")
next_prompt = self.simple_random_choose_prompt
record_prompt_cache_miss
else
record_prompt_cache_hit
end
self.delay.add_prompt_to_queue
return next_prompt
else
#Standard choose prompt at random
return self.simple_random_choose_prompt
end

end

# This prompt selection algorithm will show a visitor all combinations
# (disregarding order of display) of choices before showing repeats. Once
# all combinations have been shown, a "round" has been completed and another
# round will begin, showing all combinations again without repeats in that
# round.
def all_combos_prompt_selection(visitor)
active_choice_ids = choices.find(:all, :select => 'id', :conditions => { :active => true }).map { |c| c.id }
choice_counts = Hash[active_choice_ids.product([0])]
visitor_prompts = visitor.prompts.find(:all, :conditions => { :question_id => self })

sample_least = lambda do |sorted_choice_counts|
least = nil
choice_ids = []
sorted_choice_counts.each do |choice_id, count|
if least == nil or least == count
choice_ids << choice_id
else
record_prompt_cache_hit
break
end
self.delay.add_prompt_to_queue
return next_prompt
else
#Standard choose prompt at random
return self.simple_random_choose_prompt
least = count
end
choice_ids.sample
end

visitor_prompts.each do |p|
['left_choice_id', 'right_choice_id'].each do |choice_direction|
choice_counts[p[choice_direction]] += 1 if choice_counts.has_key?(p[choice_direction])
end
end
least_seen_choice = sample_least.call(choice_counts.sort_by {|key, value| value})
least_seen_pair_counts = Hash[active_choice_ids.product([0])]
least_seen_pair_counts.delete(least_seen_choice)
visitor_prompts.each do |p|
if p.left_choice_id == least_seen_choice
least_seen_pair_counts[p.right_choice_id] += 1
elsif p.right_choice_id == least_seen_choice
least_seen_pair_counts[p.left_choice_id] += 1
end
end

least_seen_pair = sample_least.call(least_seen_pair_counts.sort_by {|key, value| value})
prompt = prompts.find_or_initialize_by_left_choice_id_and_right_choice_id(least_seen_choice, least_seen_pair)
prompt.save
prompt.algorithm = {:name => 'all-combos'}
prompt
end

#TODO: generalize for prompts of rank > 2
Expand All @@ -117,7 +164,7 @@ def catchup_choose_prompt(num=1000)
num.times do
prompt = nil
until prompt && prompt.active?
target = rand
target = rand
left_choice_id = right_choice_id = nil

weighted.each do |item, weight|
Expand Down Expand Up @@ -157,7 +204,7 @@ def catchup_prompts_weights(tau=0.05, alpha=1)
sum += value
end

# This will not run once all prompts have been generated,
# This will not run once all prompts have been generated,
# but it prevents us from having to pregenerate all possible prompts
if weights.size < active_choices.size ** 2 - active_choices.size
active_choices.each do |l|
Expand Down Expand Up @@ -186,7 +233,7 @@ def get_optional_information(params)
current_user = self.site

if params[:with_prompt]

if params[:with_appearance] && visitor_identifier.present?
visitor = current_user.visitors.find_or_create_by_identifier(visitor_identifier)

Expand All @@ -208,7 +255,7 @@ def get_optional_information(params)
["text", "id"].each do |param|
choice = (side == "left") ? @future_prompt.left_choice : @future_prompt.right_choice
param_val = (param == "text") ? choice.data : choice.id

result.merge!({"future_#{side}_choice_#{param}_#{offset}".to_sym => param_val})
end
end
Expand All @@ -220,25 +267,25 @@ def get_optional_information(params)
result.merge!({:appearance_id => @appearance.lookup})
else
# throw some error
end
if !@prompt
end

if !@prompt
@prompt = choose_prompt(:algorithm => params[:algorithm])
end
result.merge!({:picked_prompt_id => @prompt.id})
end
end

if params[:with_visitor_stats]
visitor = current_user.visitors.find_or_create_by_identifier(visitor_identifier)
result.merge!(:visitor_votes => Vote.find_without_default_scope(:all, :conditions => {:voter_id => visitor, :question_id => self.id}).length)
result.merge!(:visitor_ideas => visitor.choices.count)
end

# this might get cpu intensive if used too often. If so, store the calculated value in redis
# and expire after X minutes
if params[:with_average_votes]
votes_by_visitors = self.votes.count(:group => 'voter_id')

if votes_by_visitors.size > 0
average = votes_by_visitors.inject(0){|total, (k,v)| total = total + v}.to_f / votes_by_visitors.size.to_f
else
Expand All @@ -260,13 +307,13 @@ def normalize!(weighted, sum=nil)
end
sum = sum.to_f
end
weighted.each do |item, weight|
weighted[item] = weight/sum
weighted.each do |item, weight|
weighted[item] = weight/sum
weighted[item] = 0.0 unless weighted[item].finite?
end
elsif weighted.instance_of?(Array)
sum = weighted.inject(0) {|sum, item| sum += item} if sum.nil?
weighted.each_with_index do |item, i|
weighted.each_with_index do |item, i|
weighted[i] = item/sum
weighted[i] = 0.0 unless weighted[i].finite?
end
Expand All @@ -286,7 +333,7 @@ def bradley_terry_probs
the_prompts = prompts_hash_by_choice_ids

# Initial probabilities chosen at random
the_choices.size.times do
the_choices.size.times do
probs << rand
prev_probs << rand
end
Expand All @@ -295,7 +342,7 @@ def bradley_terry_probs
probs_size = probs.size

difference = 1

# probably want to add a fuzz here to account for floating rounding
while difference > fuzz do
s = t % probs_size
Expand All @@ -316,7 +363,7 @@ def bradley_terry_probs

denominator+= (wins_and_losses).to_f / (prev_probs[s] + prev_probs[index])
end
probs[s] = numerator / denominator
probs[s] = numerator / denominator
# avoid divide by zero NaN
probs[s] = 0.0 unless probs[s].finite?
normalize!(probs)
Expand All @@ -328,15 +375,15 @@ def bradley_terry_probs
end
puts difference
end

probs_hash = {}
probs.each_with_index do |item, index|
probs.each_with_index do |item, index|
probs_hash[the_choices[index].id] = item
end
probs_hash
end


def all_bt_scores
btprobs = bradley_terry_probs
btprobs.each do |key, value|
Expand All @@ -354,7 +401,7 @@ def prompts_hash_by_choice_ids
the_prompts
end


def distinct_array_of_choice_ids(params={})
params = {
:rank => 2,
Expand All @@ -363,18 +410,18 @@ def distinct_array_of_choice_ids(params={})
rank = params[:rank]
only_active = params[:only_active]
count = (only_active) ? choices.active.count : choices.count

found_choices = []
# select only active choices?
conditions = (only_active) ? ['active = ?', true] : ['1=1']

rank.times do
rank.times do
# if we've already found some, make sure we don't find them again
if found_choices.count > 0
conditions[0] += ' AND id NOT IN (?)'
conditions.push found_choices
end

found_choices.push choices.find(:first,
:select => 'id',
:conditions => conditions,
Expand All @@ -383,26 +430,26 @@ def distinct_array_of_choice_ids(params={})
end
return found_choices
end

def picked_prompt_id
simple_random_choose_prompt.id
end

def self.voted_on_by(u)
select {|z| z.voted_on_by_user?(u)}
end

def voted_on_by_user?(u)
u.questions_voted_on.include? self
end

def should_autoactivate_ideas?
it_should_autoactivate_ideas?
end

validates_presence_of :site, :on => :create, :message => "can't be blank"
validates_presence_of :creator, :on => :create, :message => "can't be blank"

def density
# slow code, only to be run by cron job once at night

Expand All @@ -414,7 +461,7 @@ def density

nonseed_seed_sum= 0
nonseed_seed_total= 0

nonseed_nonseed_sum= 0
nonseed_nonseed_total= 0

Expand Down Expand Up @@ -461,7 +508,7 @@ def density
densities[:seed_nonseed] = seed_nonseed_sum.to_f / seed_nonseed_total.to_f
densities[:nonseed_seed] = nonseed_seed_sum.to_f / nonseed_seed_total.to_f
densities[:nonseed_nonseed] = nonseed_nonseed_sum.to_f / nonseed_nonseed_total.to_f

puts "Seed_seed sum: #{seed_seed_sum}, seed_seed total num: #{seed_seed_total}"
puts "Seed_nonseed sum: #{seed_nonseed_sum}, seed_nonseed total num: #{seed_nonseed_total}"
puts "Nonseed_seed sum: #{nonseed_seed_sum}, nonseed_seed total num: #{nonseed_seed_total}"
Expand Down Expand Up @@ -492,7 +539,7 @@ def clear_prompt_queue
$redis.del(self.pq_key)
end


# make prompt queue less than @@precent_full
def mark_prompt_queue_for_refill
# 2 because redis starts indexes at 0
Expand Down Expand Up @@ -575,12 +622,12 @@ def to_csv(type)
when 'votes'

headers = ['Vote ID', 'Session ID', 'Wikisurvey ID','Winner ID', 'Winner Text', 'Loser ID', 'Loser Text', 'Prompt ID', 'Appearance ID', 'Left Choice ID', 'Right Choice ID', 'Created at', 'Updated at', 'Response Time (s)', 'Missing Response Time Explanation', 'Session Identifier', 'Valid']

when 'ideas'
headers = ['Wikisurvey ID','Idea ID', 'Idea Text', 'Wins', 'Losses', 'Times involved in Cant Decide', 'Score', 'User Submitted', 'Session ID', 'Created at', 'Last Activity', 'Active', 'Appearances on Left', 'Appearances on Right', 'Session Identifier']
when 'non_votes'
headers = ['Record Type', 'Skip ID', 'Appearance ID', 'Session ID', 'Wikisurvey ID','Left Choice ID', 'Left Choice Text', 'Right Choice ID', 'Right Choice Text', 'Prompt ID', 'Reason', 'Created at', 'Updated at', 'Response Time (s)', 'Missing Response Time Explanation', 'Session Identifier', 'Valid']
else
else
raise "Unsupported export type: #{type}"
end

Expand Down Expand Up @@ -623,17 +670,17 @@ def to_csv(type)

end
when 'non_votes'

self.appearances.find_each(:include => [:voter], :conditions => ['answerable_type <> ? OR answerable_type IS NULL', 'Vote']) do |a|

if a.answerable_type == 'Skip'
# If this appearance belongs to a skip, show information on the skip instead
s = a.answerable
valid = s.valid_record ? 'TRUE' : 'FALSE'
time_viewed = s.time_viewed.nil? ? "NA": s.time_viewed.to_f / 1000.0
prompt = s.prompt
y.yield [ "Skip", s.id, a.id, s.skipper_id, s.question_id, s.prompt.left_choice.id, s.prompt.left_choice.data.strip, s.prompt.right_choice.id, s.prompt.right_choice.data.strip, s.prompt_id, s.skip_reason, s.created_at, s.updated_at, time_viewed , s.missing_response_time_exp, s.skipper.identifier,valid].to_csv

else
# If no skip and no vote, this is an orphaned appearance
prompt = a.prompt
Expand Down Expand Up @@ -704,7 +751,7 @@ def create_or_find_next_appearance(visitor, params, offset=0)
# Only choose prompt if we don't already have one. If we had to
# retry this transaction due to a deadlock, a prompt may have been
# selected previously.
prompt = choose_prompt(:algorithm => params[:algorithm]) unless prompt
prompt = choose_prompt(:algorithm => params[:algorithm], :visitor => visitor) unless prompt
appearance = self.site.record_appearance(visitor, prompt)
end
end
Expand Down
Loading