Add item bank to power_summary of RecordedCatLoop

* Clear recorded data on run_cat startup

Author: Frankie Robertson

Project.toml

@@ -50,7 +50,7 @@ DocStringExtensions = " ^0.9"
 EffectSizes = "^1.0.1"
 ElasticArrays = "1.2.12"
 FillArrays = "0.13, 1.5.0"
-FittedItemBanks = "^0.7.2"
+FittedItemBanks = "^0.7.3"
 ForwardDiff = "1"
 HypothesisTests = "^0.10.12, ^0.11.0"
 Interpolations = "^0.14, ^0.15"
@@ -62,7 +62,7 @@ MacroTools = "^0.5.6"
 Mmap = "^1.11"
 Optim = "1.7.3"
 PrecompileTools = "1.2.1"
-PsychometricsBazaarBase = "^0.8.4"
+PsychometricsBazaarBase = "^0.8.6"
 QuickHeaps = "0.2.2"
 Random = "^1.11"
 Reexport = "1"

src/Aggregators/Aggregators.jl

@@ -20,6 +20,7 @@ using FittedItemBanks: AbstractItemBank, ContinuousDomain,
 using ..Responses
 using ..Responses: concrete_response_type, function_xs, function_ys, Responses
 using ..ConfigBase
+using PsychometricsBazaarBase: power_summary
 using PsychometricsBazaarBase.ConfigTools: @requiresome, @returnsome,
                                            find1_instance, find1_type,
                                            find1_type_sloppy

src/Aggregators/ability_estimator.jl

@@ -65,7 +65,7 @@ function show(io::IO, ::MIME"text/plain", ability_estimator::PosteriorAbilityEst
     println(io, "Ability posterior distribution")
     indent_io = indent(io, 2)
     print(indent_io, "Prior: ")
-    show(indent_io, MIME("text/plain"), ability_estimator.prior)
+    power_summary(indent_io, ability_estimator.prior)
     println(io)
 end
 

src/Rules.jl

@@ -12,6 +12,7 @@ using ..NextItemRules: NextItemRule
 using ..TerminationConditions: TerminationCondition
 using ..ConfigBase
 import Base: show
+import PsychometricsBazaarBase: power_summary
 
 """
 $(TYPEDEF)
@@ -82,12 +83,21 @@ function CatRules(bits...)
 end
 
 function show(io::IO, ::MIME"text/plain", rules::CatRules)
+    power_summary(io, rules; toplevel=true)
+end
+
+function power_summary(io::IO, rules::CatRules; toplevel=false)
+    # TODO
     print(io, "Next item rule: ")
     show(io, MIME("text/plain"), rules.next_item)
-    println(io)
+    if toplevel
+        println(io)
+    end
     print(io, "Termination condition: ")
     show(io, MIME("text/plain"), rules.termination_condition)
-    println(io)
+    if toplevel
+        println(io)
+    end
     print(io, "Ability estimator: ")
     show(io, MIME("text/plain"), rules.ability_estimator)
 end

src/Sim/Sim.jl

@@ -2,10 +2,11 @@ module Sim
 
 using DataFrames: DataFrame
 using ElasticArrays
-using ElasticArrays: sizehint_lastdim!
+using ElasticArrays: sizehint_lastdim!, resize_lastdim!
 using DocStringExtensions
 using StatsBase
 using FittedItemBanks: AbstractItemBank, ResponseType, ItemResponse, domdims
+using PsychometricsBazaarBase: show_into_buf, power_summary_into_buf
 using PsychometricsBazaarBase.Integrators
 using PsychometricsBazaarBase.IndentWrappers: indent
 using ..ConfigBase
@@ -25,6 +26,7 @@ using ..Aggregators: TrackedResponses,
                      RiemannEnumerationIntegrator
 using ..NextItemRules: AbilityVariance, compute_criteria, best_item
 import Base: show
+import PsychometricsBazaarBase: power_summary
 
 export CatRecorder, CatRecording
 export CatLoop, record!

src/Sim/loop.jl

@@ -24,40 +24,48 @@ struct CatLoop{CatEngineT} <: CatConfigBase
     A callback called each time there is a new responses.
     If provided, it is passed `(responses::TrackedResponses, terminating)`.
     """
-    new_response_callback
+    new_response_callback::Any
+    """
+    A callback called each time a CAT is run
+    If provided, it is passed `(item_bank::AbstractItemBank)`.
+    """
+    init_callback::Any
 end
 
 function show(io::IO, ::MIME"text/plain", rules::CatLoop)
-    print(io, "Next item rule: ")
-    show(io, MIME("text/plain"), rules.next_item)
-    print(io, "Termination condition: ")
-    show(io, MIME("text/plain"), rules.termination_condition)
-    print(io, "Ability estimator: ")
-    show(io, MIME("text/plain"), rules.ability_estimator)
+    print(io, "Computer-Adaptive Test Loop based on ")
+    show(io, MIME("text/plain"), rules.rules)
+end
+
+function collate_cat_callbacks(callbacks...)
+    callbacks = filter(!isnothing, callbacks)
+    function all_callbacks(args...)
+        for callback in callbacks
+            callback(args...)
+        end
+        nothing
+    end
+    all_callbacks
 end
 
 function CatLoop(;
     rules,
     get_response,
     new_response_callback = nothing,
     new_response_callbacks = Any[],
+    init_callback = nothing,
+    init_callbacks = Any[],
     recorder = nothing
 )
-    new_response_callbacks = collect(new_response_callbacks)
-    if new_response_callback !== nothing
-        push!(new_response_callbacks, new_response_callback)
-    end
-    if recorder !== nothing && showable(MIME("text/plain"), rules)
-        buf = IOBuffer()
-        show(buf, MIME("text/plain"), rules)
-        recorder.recording.rules_description = String(take!(buf))
-        push!(new_response_callbacks, catrecorder_callback(recorder))
-    end
-    function all_callbacks(responses, terminating)
-        for callback in new_response_callbacks
-            callback(responses, terminating)
-        end
-        nothing
-    end
-    CatLoop{typeof(rules)}(rules, get_response, all_callbacks)
-end
+    new_response_callback = collate_cat_callbacks(
+        new_response_callbacks...,
+        new_response_callback,
+        isnothing(recorder) ? nothing : recorder_response_callback(recorder)
+    )
+    init_callback = collate_cat_callbacks(
+        init_callbacks...,
+        init_callback,
+        isnothing(recorder) ? nothing : recorder_init_callback(recorder)
+    )
+    CatLoop{typeof(rules)}(rules, get_response, new_response_callback, init_callback)
+end

src/Sim/recorded_loop.jl

@@ -191,3 +191,8 @@ function run_cat(loop::RecordedCatLoop; ib_labels = nothing)
     end
     run_cat(loop, loop.item_bank; ib_labels=ib_labels)
 end
+
+function show(io::IO, ::MIME"text/plain", loop::RecordedCatLoop)
+    println(io, "Recorded Computer-Adaptive Test:")
+    power_summary(io, loop.recorder.recording; skip_first_line=true)
+end

src/Sim/recorder.jl

@@ -22,7 +22,8 @@ Base.@kwdef mutable struct CatRecording{LikelihoodsT <: NamedTuple}
     data::LikelihoodsT
     item_index::Vector{Int}
     item_correctness::Vector{Bool}
-    rules_description::Union{Nothing, String} = nothing
+    rules_description::Union{Nothing, IOBuffer} = nothing
+    item_bank_description::Union{Nothing, IOBuffer} = nothing
 end
 
 function Base.getproperty(obj::CatRecording, sym::Symbol)
@@ -67,26 +68,33 @@ function prepare_dataframe(recording::CatRecording)
 end
 
 function show(io::IO, ::MIME"text/plain", recording::CatRecording)
-    println(io, "Recording of a Computer-Adaptive Test")
-    if recording.rules_description === nothing
+    power_summary(io, recording; include_cat_config = :always)
+end
+
+function power_summary(io::IO, recording::CatRecording; include_cat_config = :always, skip_first_line=false, kwargs...)
+    if !skip_first_line
+        println(io, "Recording of a Computer-Adaptive Test")
+    end
+    if recording.rules_description === nothing && include_cat_config == :always
         println(io, "  Unknown CAT configuration")
-    else
+    elseif include_cat_config != :never # :available or :always
         println(io, "  CAT configuration:")
-        for line in split(strip(recording.rules_description, '\n'), "\n")
-            println(io, "    ", line)
-        end
+        write(indent(io, 4), recording.rules_description)
+        seekstart(recording.rules_description)
+        println(io)
+    end
+    if recording.item_bank_description === nothing
+        println(io, "  Unknown item bank")
+    else
+        println(io, "  Item bank:")
+        write(indent(io, 4), recording.item_bank_description)
+        seekstart(recording.item_bank_description)
+        println(io)
     end
-    println(io)
     println(io, "  Recorded information:")
     df = prepare_dataframe(recording)
-    buf = IOBuffer()
-    show(buf, MIME("text/plain"), df; summary=false, eltypes=false, rowlabel=:Number)
-    seekstart(buf)
-    for line in eachline(buf)
-        println(io, "    ", line)
-    end
-    #println(io)
-    #println(io, "  Final information:")
+    buf = show_into_buf(df; summary = false, eltypes = false, rowlabel = :Number)
+    write(indent(io, 4), buf)
 end
 
 #=
@@ -132,6 +140,18 @@ function record!(recording::CatRecording, responses; data...)
     push!(recording.item_correctness, item_correct)
 end
 
+function Base.empty!(recording::CatRecording)
+    empty!(recording.item_index)
+    empty!(recording.item_correctness)
+    for (name, value) in pairs(recording.data)
+        if value.data isa AbstractVector
+            empty!(value.data)
+        elseif value.data isa ElasticArray
+            resize_lastdim!(value.data, 0)
+        end
+    end
+end
+
 #=
 """
 $(TYPEDSIGNATURES)
@@ -402,6 +422,27 @@ function record!(recorder::CatRecorder, tracked_responses)
     record!(recorder.recording, tracked_responses.responses)
 end
 
-function catrecorder_callback(recoder::CatRecorder)
-    return (tracked_responses, _) -> record!(recoder, tracked_responses)
+function recorder_response_callback(recorder::CatRecorder)
+    return (tracked_responses, _) -> record!(recorder, tracked_responses)
+end
+
+function recorder_init_callback(recorder::CatRecorder)
+    return function (cat_loop, item_bank)
+        empty!(recorder.recording)
+        if showable(MIME("text/plain"), cat_loop.rules)
+            recorder.recording.rules_description = power_summary_into_buf(cat_loop.rules; toplevel=false)
+        end
+        if showable(MIME("text/plain"), item_bank)
+            recorder.recording.item_bank_description = power_summary_into_buf(item_bank)
+        end
+    end
+end
+
+function show(io::IO, ::MIME"text/plain", recorder::CatRecorder)
+    indent_io = indent(io, 4)
+    println(io, "Computer-Adaptive Test Recorder")
+    println(io, "  Requests:")
+    show(indent_io, MIME"text/plain", recorder.requests)
+    println(io, "  Recording:")
+    show(indent_io, MIME"text/plain", recorder.recording)
 end

src/Sim/run.jl

@@ -43,7 +43,10 @@ If `ib_labels` is not given, default labels of the form
 function run_cat(loop::CatLoop{RulesT},
         item_bank::AbstractItemBank;
         ib_labels = nothing) where {RulesT <: CatRules}
-    (; rules, get_response, new_response_callback) = loop
+    (; rules, get_response, new_response_callback, init_callback) = loop
+    if init_callback !== nothing
+        init_callback(loop, item_bank)
+    end
     (; next_item, termination_condition, ability_estimator, ability_tracker) = rules
     responses = TrackedResponses(BareResponses(ResponseType(item_bank)),
         item_bank,
@@ -81,4 +84,4 @@ function run_cat(loop::CatLoop{RulesT},
         end
     end
     (responses.responses, ability_estimator(responses))
-end
+end