first commit

question/behaviour/deferredcbm/behaviour.php

@@ -0,0 +1,124 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Question behaviour that is like the deferred feedback model, but with
+ * certainty based marking. That is, in addition to the other controls, there are
+ * where the student can indicate how certain they are that their answer is right.
+ *
+ * @package    qbehaviour
+ * @subpackage deferredcbm
+ * @copyright  2009 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+
+defined('MOODLE_INTERNAL') || die();
+
+require_once(__DIR__ . '/../deferredfeedback/behaviour.php');
+
+
+/**
+ * Question behaviour for deferred feedback with certainty based marking.
+ *
+ * The student enters their response during the attempt, along with a certainty,
+ * that is, how sure they are that they are right, and it is saved. Later,
+ * when the whole attempt is finished, their answer is graded. Their degree
+ * of certainty affects their score.
+ *
+ * @copyright  2009 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class qbehaviour_deferredcbm extends qbehaviour_deferredfeedback {
+    const IS_ARCHETYPAL = true;
+
+    public function get_min_fraction() {
+        return question_cbm::adjust_fraction(0, question_cbm::HIGH);
+    }
+
+    public function get_max_fraction() {
+        return question_cbm::adjust_fraction(1, question_cbm::HIGH);
+    }
+
+    public function get_expected_data() {
+        if ($this->qa->get_state()->is_active()) {
+            return array('certainty' => PARAM_INT);
+        }
+        return parent::get_expected_data();
+    }
+
+    public function get_right_answer_summary() {
+        $summary = parent::get_right_answer_summary();
+        return question_cbm::summary_with_certainty($summary, question_cbm::HIGH);
+    }
+
+    public function get_correct_response() {
+        if ($this->qa->get_state()->is_active()) {
+            return array('certainty' => question_cbm::HIGH);
+        }
+        return array();
+    }
+
+    protected function get_our_resume_data() {
+        $lastcertainty = $this->qa->get_last_behaviour_var('certainty');
+        if ($lastcertainty) {
+            return array('-certainty' => $lastcertainty);
+        } else {
+            return array();
+        }
+    }
+
+    protected function is_same_response(question_attempt_step $pendingstep) {
+        return parent::is_same_response($pendingstep) &&
+                $this->qa->get_last_behaviour_var('certainty') ==
+                        $pendingstep->get_behaviour_var('certainty');
+    }
+
+    protected function is_complete_response(question_attempt_step $pendingstep) {
+        return parent::is_complete_response($pendingstep) &&
+                $pendingstep->has_behaviour_var('certainty');
+    }
+
+    public function process_finish(question_attempt_pending_step $pendingstep) {
+        $status = parent::process_finish($pendingstep);
+        if ($status == question_attempt::KEEP) {
+            $fraction = $pendingstep->get_fraction();
+            if ($this->qa->get_last_step()->has_behaviour_var('certainty')) {
+                $certainty = $this->qa->get_last_step()->get_behaviour_var('certainty');
+            } else {
+                $certainty = question_cbm::default_certainty();
+                $pendingstep->set_behaviour_var('_assumedcertainty', $certainty);
+            }
+            if (!is_null($fraction)) {
+                $pendingstep->set_behaviour_var('_rawfraction', $fraction);
+                $pendingstep->set_fraction(question_cbm::adjust_fraction($fraction, $certainty));
+            }
+            $pendingstep->set_new_response_summary(
+                    question_cbm::summary_with_certainty($pendingstep->get_new_response_summary(),
+                    $this->qa->get_last_step()->get_behaviour_var('certainty')));
+        }
+        return $status;
+    }
+
+    public function summarise_action(question_attempt_step $step) {
+        $summary = parent::summarise_action($step);
+        if ($step->has_behaviour_var('certainty')) {
+            $summary = question_cbm::summary_with_certainty($summary,
+                    $step->get_behaviour_var('certainty'));
+        }
+        return $summary;
+    }
+}

question/behaviour/deferredcbm/behaviourtype.php

@@ -0,0 +1,249 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Question behaviour type for deferred feedback with CBM behaviour.
+ *
+ * @package    qbehaviour_deferredcbm
+ * @copyright  2012 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+
+defined('MOODLE_INTERNAL') || die();
+
+require_once(__DIR__ . '/../deferredfeedback/behaviourtype.php');
+
+
+/**
+ * Question behaviour type information for deferred feedback with CBM behaviour.
+ *
+ * @copyright  2012 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class qbehaviour_deferredcbm_type extends qbehaviour_deferredfeedback_type {
+    public function adjust_random_guess_score($fraction) {
+        return question_cbm::adjust_fraction($fraction, question_cbm::default_certainty());
+    }
+
+    public function summarise_usage(question_usage_by_activity $quba, question_display_options $options) {
+        global $OUTPUT;
+        $summarydata = parent::summarise_usage($quba, $options);
+
+        if ($options->marks < question_display_options::MARK_AND_MAX) {
+            return $summarydata;
+        }
+
+        // Prepare accumulators to hold the data we are about to collect.
+        $notansweredcount  = 0;
+        $notansweredweight = 0;
+        $attemptcount = array(
+            question_cbm::HIGH => 0,
+            question_cbm::MED  => 0,
+            question_cbm::LOW  => 0,
+        );
+        $totalweight = array(
+            question_cbm::HIGH => 0,
+            question_cbm::MED  => 0,
+            question_cbm::LOW  => 0,
+        );
+        $totalrawscore = array(
+            question_cbm::HIGH => 0,
+            question_cbm::MED  => 0,
+            question_cbm::LOW  => 0,
+        );
+        $totalcbmscore = array(
+            question_cbm::HIGH => 0,
+            question_cbm::MED  => 0,
+            question_cbm::LOW  => 0,
+        );
+
+        // Loop through the data, and add it to the accumulators.
+        foreach ($quba->get_attempt_iterator() as $qa) {
+            if (strpos($qa->get_behaviour_name(), 'cbm') === false || $qa->get_max_mark() < 0.0000005) {
+                continue;
+            }
+
+            $gradedstep = $qa->get_last_step_with_behaviour_var('_rawfraction');
+
+            if (!$gradedstep->has_behaviour_var('_rawfraction')) {
+                $notansweredcount  += 1;
+                $notansweredweight += $qa->get_max_mark();
+                continue;
+            }
+
+            $certainty = $qa->get_last_behaviour_var('certainty');
+            if (is_null($certainty) || $certainty == -1) {
+                // Certainty -1 has never been used in standard Moodle, but is
+                // used in Tony-Gardiner Medwin's patches to mean 'No idea' which
+                // we intend to implement: MDL-42077. In the mean time, avoid
+                // errors for people who have used TGM's patches.
+                $certainty = question_cbm::default_certainty();
+            }
+
+            $attemptcount[$certainty]  += 1;
+            $totalweight[$certainty]   += $qa->get_max_mark();
+            $totalrawscore[$certainty] += $qa->get_max_mark() * $gradedstep->get_behaviour_var('_rawfraction');
+            $totalcbmscore[$certainty] += $qa->get_mark();
+        }
+
+        // Hence compute some statistics.
+        $totalquestions   = $notansweredcount + array_sum($attemptcount);
+        $grandtotalweight = $notansweredweight + array_sum($totalweight);
+        $accuracy         = array_sum($totalrawscore) / $grandtotalweight;
+        $averagecbm       = array_sum($totalcbmscore) / $grandtotalweight;
+        $cbmbonus         = $this->calculate_bonus($averagecbm, $accuracy);
+        $accuracyandbonus = $accuracy + $cbmbonus;
+
+        // Add a note to explain the max mark.
+        $summarydata['qbehaviour_cbm_grade_explanation'] = array(
+            'title' => '',
+            'content' => html_writer::tag('i', get_string('cbmgradeexplanation', 'qbehaviour_deferredcbm')) .
+                    $OUTPUT->help_icon('cbmgrades', 'qbehaviour_deferredcbm'),
+        );
+
+        // Now we can start generating some of the summary: overall values.
+        $summarydata['qbehaviour_cbm_entire_quiz_heading'] = array(
+            'title' => '',
+            'content' => html_writer::tag('h3',
+                    get_string('forentirequiz', 'qbehaviour_deferredcbm', $totalquestions),
+                    array('class' => 'qbehaviour_deferredcbm_summary_heading')),
+        );
+        $summarydata['qbehaviour_cbm_entire_quiz_cbm_average'] = array(
+            'title' => get_string('averagecbmmark', 'qbehaviour_deferredcbm'),
+            'content' => format_float($averagecbm, $options->markdp),
+        );
+        $summarydata['qbehaviour_cbm_entire_quiz_accuracy'] = array(
+            'title' => get_string('accuracy', 'qbehaviour_deferredcbm'),
+            'content' => $this->format_probability($accuracy, 1),
+        );
+        $summarydata['qbehaviour_cbm_entire_quiz_cbm_bonus'] = array(
+            'title' => get_string('cbmbonus', 'qbehaviour_deferredcbm'),
+            'content' => $this->format_probability($cbmbonus, 1),
+        );
+        $summarydata['qbehaviour_cbm_entire_quiz_accuracy_and_bonus'] = array(
+            'title' => get_string('accuracyandbonus', 'qbehaviour_deferredcbm'),
+            'content' => $this->format_probability($accuracyandbonus, 1),
+        );
+
+        if ($notansweredcount && array_sum($attemptcount) > 0) {
+            $totalquestions   = array_sum($attemptcount);
+            $grandtotalweight = array_sum($totalweight);
+            $accuracy         = array_sum($totalrawscore) / $grandtotalweight;
+            $averagecbm       = array_sum($totalcbmscore) / $grandtotalweight;
+            $cbmbonus         = $this->calculate_bonus($averagecbm, $accuracy);
+            $accuracyandbonus = $accuracy + $cbmbonus;
+
+            $summarydata['qbehaviour_cbm_answered_quiz_heading'] = array(
+                'title' => '',
+                'content' => html_writer::tag('h3',
+                        get_string('foransweredquestions', 'qbehaviour_deferredcbm', $totalquestions),
+                        array('class' => 'qbehaviour_deferredcbm_summary_heading')),
+            );
+            $summarydata['qbehaviour_cbm_answered_quiz_cbm_average'] = array(
+                'title' => get_string('averagecbmmark', 'qbehaviour_deferredcbm'),
+                'content' => format_float($averagecbm, $options->markdp),
+            );
+            $summarydata['qbehaviour_cbm_answered_quiz_accuracy'] = array(
+                'title' => get_string('accuracy', 'qbehaviour_deferredcbm'),
+                'content' => $this->format_probability($accuracy, 1),
+            );
+            $summarydata['qbehaviour_cbm_answered_quiz_cbm_bonus'] = array(
+                'title' => get_string('cbmbonus', 'qbehaviour_deferredcbm'),
+                'content' => $this->format_probability($cbmbonus, 1),
+            );
+            $summarydata['qbehaviour_cbm_answered_quiz_accuracy_and_bonus'] = array(
+                'title' => get_string('accuracyandbonus', 'qbehaviour_deferredcbm'),
+                'content' => $this->format_probability($accuracyandbonus, 1),
+            );
+        }
+
+        // Now per-certainty level values.
+        $summarydata['qbehaviour_cbm_judgement_heading'] = array(
+            'title' => '',
+            'content' => html_writer::tag('h3', get_string('breakdownbycertainty', 'qbehaviour_deferredcbm'),
+                    array('class' => 'qbehaviour_deferredcbm_summary_heading')),
+        );
+
+        foreach ($attemptcount as $certainty => $count) {
+            $key   = 'qbehaviour_cbm_judgement' . $certainty;
+            $title = question_cbm::get_short_string($certainty);
+
+            if ($count == 0) {
+                $summarydata[$key] = array(
+                    'title' => $title,
+                    'content' => get_string('noquestions', 'qbehaviour_deferredcbm'),
+                );
+                continue;
+            }
+
+            $lowerlimit = question_cbm::optimal_probablility_low($certainty);
+            $upperlimit = question_cbm::optimal_probablility_high($certainty);
+            $fraction = $totalrawscore[$certainty] / $totalweight[$certainty];
+
+            $a = new stdClass();
+            $a->responses = $count;
+            $a->idealrangelow  = $this->format_probability($lowerlimit);
+            $a->idealrangehigh = $this->format_probability($upperlimit);
+            $a->fraction       = html_writer::tag('span', $this->format_probability($fraction),
+                    array('class' => 'qbehaviour_deferredcbm_actual_percentage'));
+
+            if ($fraction < $lowerlimit - 0.0000005) {
+                if ((pow($fraction - $lowerlimit, 2) * $count) > 0.5) { // Rough indicator of significance: t > 1.5 or 1.8.
+                    $judgement = 'overconfident';
+                } else {
+                    $judgement = 'slightlyoverconfident';
+                }
+            } else if ($fraction > $upperlimit + 0.0000005) {
+                if ((pow($fraction - $upperlimit, 2) * $count) > 0.5) {
+                    $judgement = 'underconfident';
+                } else {
+                    $judgement = 'slightlyunderconfident';
+                }
+            } else {
+                $judgement = 'judgementok';
+            }
+            $a->judgement = html_writer::tag('span', get_string($judgement, 'qbehaviour_deferredcbm'),
+                    array('class' => 'qbehaviour_deferredcbm_' . $judgement));
+
+            $summarydata[$key] = array(
+                'title' => $title,
+                'content' => get_string('judgementsummary', 'qbehaviour_deferredcbm', $a),
+            );
+        }
+
+        return $summarydata;
+    }
+
+    protected function format_probability($probability, $dp = 0) {
+        return format_float($probability * 100, $dp) . '%';
+    }
+
+    public function calculate_bonus($total, $accuracy) {
+        $expectedforaccuracy = max(
+            $accuracy * question_cbm::adjust_fraction(1, question_cbm::LOW) +
+                (1 - $accuracy) * question_cbm::adjust_fraction(0, question_cbm::LOW),
+            $accuracy * question_cbm::adjust_fraction(1, question_cbm::MED) +
+                (1 - $accuracy) * question_cbm::adjust_fraction(0, question_cbm::MED),
+            $accuracy * question_cbm::adjust_fraction(1, question_cbm::HIGH) +
+                (1 - $accuracy) * question_cbm::adjust_fraction(0, question_cbm::HIGH)
+        );
+        // The constant 0.1 here is determinted empirically from looking at lots
+        // for CBM quiz results. See www.ucl.ac.uk/~ucgbarg/tea/IUPS_2013a.pdf.
+        // It approximately maximises the reliability of accuracy + bonus.
+        return 0.1 * ($total - $expectedforaccuracy);
+    }
+}

question/behaviour/deferredcbm/classes/privacy/provider.php

@@ -0,0 +1,46 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Privacy Subsystem implementation for qbehaviour_deferredcbm.
+ *
+ * @package    qbehaviour_deferredcbm
+ * @copyright  2018 Andrew Nicols <andrew@nicols.co.uk>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace qbehaviour_deferredcbm\privacy;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Privacy Subsystem for qbehaviour_deferredcbm implementing null_provider.
+ *
+ * @copyright  2018 Andrew Nicols <andrew@nicols.co.uk>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class provider implements \core_privacy\local\metadata\null_provider {
+
+    /**
+     * Get the language string identifier with the component's language
+     * file to explain why this plugin stores no data.
+     *
+     * @return  string
+     */
+    public static function get_reason(): string {
+        return 'privacy:metadata';
+    }
+}

question/behaviour/deferredcbm/lang/en/qbehaviour_deferredcbm.php

@@ -0,0 +1,73 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Strings for component 'qbehaviour_deferredcbm', language 'en'.
+ *
+ * @package    qbehaviour
+ * @subpackage deferredcbm
+ * @copyright  2009 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+$string['accuracy'] = 'Accuracy';
+$string['accuracyandbonus'] = 'Accuracy + Bonus';
+$string['assumingcertainty'] = 'You did not select a certainty. Assuming: {$a}.';
+$string['averagecbmmark'] = 'Average CBM mark';
+$string['basemark'] = 'Base mark {$a}';
+$string['breakdownbycertainty'] = 'Break-down by certainty';
+$string['cbmbonus'] = 'CBM bonus';
+$string['cbmmark'] = 'CBM mark {$a}';
+$string['cbmgradeexplanation'] = 'For CBM, the grade above is shown relative to the maximum for all correct at C=1.';
+$string['cbmgrades'] = 'CBM grades';
+$string['cbmgrades_help'] = 'With Certainty Based Marking (CBM) getting every question correct with C=1 (low certainty) gives a grade of 100%. Grades may be as high as 300% if every question is correct with C=3 (high certainty). Misconceptions (confident wrong responses) lower grades much more than wrong responses that are acknowledged to be uncertain. This may even lead to negative overall grades.
+
+**Accuracy** is the % correct ignoring certainty but weighted for the maximum mark of each question. Successfully distinguishing more and less reliable responses gives a better grade than selecting the same certainty for each question. This is reflected in the **CBM Bonus**. **Accuracy** + **CBM Bonus** is a better measure of knowledge than **Accuracy**. Misconceptions can lead to a negative bonus, a warning to look carefully at what is and is not known.';
+$string['cbmgrades_link'] = 'qbehaviour/deferredcbm/certaintygrade';
+$string['certainty'] = 'Certainty';
+$string['certainty_help'] = 'Certainty-based marking requires you to indicate how reliable you think your answer is. The available levels are:
+
+Certainty level     | C=1 (Unsure) | C=2 (Mid) | C=3 (Quite sure)
+------------------- | ------------ | --------- | ----------------
+Mark if correct     |   1          |    2      |      3
+Mark if wrong       |   0          |   -2      |     -6
+Probability correct |  <67%        | 67-80%    |    >80%
+
+Best marks are gained by acknowledging uncertainty. For example, if you think there is more than a 1 in 3 chance of being wrong, you should enter C=1 and avoid the risk of a negative mark.
+';
+$string['certainty_link'] = 'qbehaviour/deferredcbm/certainty';
+$string['certainty-1'] = 'No Idea';
+$string['certainty1'] = 'C=1 (Unsure: <67%)';
+$string['certainty2'] = 'C=2 (Mid: >67%)';
+$string['certainty3'] = 'C=3 (Quite sure: >80%)';
+$string['certaintyshort-1'] = 'No Idea';
+$string['certaintyshort1'] = 'C=1';
+$string['certaintyshort2'] = 'C=2';
+$string['certaintyshort3'] = 'C=3';
+$string['dontknow'] = 'No idea';
+$string['foransweredquestions'] = 'Results for just the {$a} answered questions';
+$string['forentirequiz'] = 'Results for the whole quiz ({$a} questions)';
+$string['judgementok'] = 'OK';
+$string['judgementsummary'] = 'Responses: {$a->responses}. Accuracy: {$a->fraction}. (Optimal range {$a->idealrangelow} to {$a->idealrangehigh}). You were {$a->judgement} using this certainty level.';
+$string['howcertainareyou'] = 'Certainty{$a->help}: {$a->choices}';
+$string['noquestions'] = 'No responses';
+$string['overconfident'] = 'over-confident';
+$string['pluginname'] = 'Deferred feedback with CBM';
+$string['privacy:metadata'] = 'The Deferred feedback with CBM question behaviour plugin does not store any personal data.';
+$string['slightlyoverconfident'] = 'a bit over-confident';
+$string['slightlyunderconfident'] = 'a bit under-confident';
+$string['underconfident'] = 'under-confident';
+$string['weightx'] = 'Weight {$a}';

question/behaviour/deferredcbm/renderer.php

@@ -0,0 +1,108 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Defines the renderer for the deferred feedback with certainty based marking
+ * behaviour.
+ *
+ * @package    qbehaviour
+ * @subpackage deferredcbm
+ * @copyright  2009 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+
+defined('MOODLE_INTERNAL') || die();
+
+
+/**
+ * Renderer for outputting parts of a question belonging to the deferred
+ * feedback with certainty based marking behaviour.
+ *
+ * @copyright  2009 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class qbehaviour_deferredcbm_renderer extends qbehaviour_renderer {
+    protected function certainty_choices($controlname, $selected, $readonly) {
+        $attributes = array(
+            'type' => 'radio',
+            'name' => $controlname,
+        );
+        if ($readonly) {
+            $attributes['disabled'] = 'disabled';
+        }
+
+        $choices = '';
+        foreach (question_cbm::$certainties as $certainty) {
+            $id = $controlname . $certainty;
+            $attributes['id'] = $id;
+            $attributes['value'] = $certainty;
+            if ($selected == $certainty) {
+                $attributes['checked'] = 'checked';
+            } else {
+                unset($attributes['checked']);
+            }
+            $choices .= ' ' .
+                    html_writer::tag('label', html_writer::empty_tag('input', $attributes) .
+                            question_cbm::get_string($certainty), array('for' => $id));
+        }
+        return $choices;
+    }
+
+    public function controls(question_attempt $qa, question_display_options $options) {
+        $a = new stdClass();
+        $a->help = $this->output->help_icon('certainty', 'qbehaviour_deferredcbm');
+        $a->choices = $this->certainty_choices($qa->get_behaviour_field_name('certainty'),
+                $qa->get_last_behaviour_var('certainty'), $options->readonly);
+
+        return html_writer::tag('div', get_string('howcertainareyou', 'qbehaviour_deferredcbm', $a),
+                array('class' => 'certaintychoices'));
+    }
+
+    public function feedback(question_attempt $qa, question_display_options $options) {
+        if (!$options->feedback) {
+            return '';
+        }
+
+        if ($qa->get_state() == question_state::$gaveup || $qa->get_state() ==
+                question_state::$mangaveup) {
+            return '';
+        }
+
+        $feedback = '';
+        if (!$qa->get_last_behaviour_var('certainty') &&
+                $qa->get_last_behaviour_var('_assumedcertainty')) {
+            $feedback .= html_writer::tag('p',
+                    get_string('assumingcertainty', 'qbehaviour_deferredcbm',
+                    question_cbm::get_string($qa->get_last_behaviour_var('_assumedcertainty'))));
+        }
+
+        return $feedback;
+    }
+
+    public function marked_out_of_max(question_attempt $qa, core_question_renderer $qoutput,
+            question_display_options $options) {
+        return get_string('weightx', 'qbehaviour_deferredcbm', $qa->format_fraction_as_mark(
+                question_cbm::adjust_fraction(1, question_cbm::default_certainty()),
+                $options->markdp));
+    }
+
+    public function mark_out_of_max(question_attempt $qa, core_question_renderer $qoutput,
+            question_display_options $options) {
+        return get_string('cbmmark', 'qbehaviour_deferredcbm', $qa->format_mark($options->markdp)) .
+                '<br>' . $this->marked_out_of_max($qa, $qoutput, $options);
+    }
+}

question/behaviour/deferredcbm/styles.css

@@ -0,0 +1,32 @@
+.qbehaviour_deferredcbm_slightlyunderconfident,
+.qbehaviour_deferredcbm_slightlyoverconfident {
+    font-weight: bold;
+    color: #600;
+}
+
+.qbehaviour_deferredcbm_underconfident,
+.qbehaviour_deferredcbm_overconfident {
+    font-weight: bold;
+    color: #c00;
+}
+
+.qbehaviour_deferredcbm_judgementok {
+    font-weight: bold;
+    color: #080;
+}
+
+.qbehaviour_deferredcbm_actual_percentage {
+    font-weight: bold;
+}
+
+.qbehaviour_deferredcbm_summary_heading {
+    margin: 0;
+}
+
+.que.deferredcbm .certaintychoices input[type="radio"] {
+    margin-left: 0.5em;
+}
+
+.que.deferredcbm .certaintychoices label {
+    white-space: nowrap;
+}

question/behaviour/deferredcbm/tests/behaviour_type_test.php

@@ -0,0 +1,142 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace qbehaviour_deferredcbm;
+
+use question_display_options;
+use question_engine;
+use question_testcase;
+
+defined('MOODLE_INTERNAL') || die();
+
+global $CFG;
+require_once(__DIR__ . '/../../../engine/lib.php');
+require_once(__DIR__ . '/../../../engine/tests/helpers.php');
+
+
+/**
+ * Unit tests for the deferred feedback with certainty base marking behaviour.
+ *
+ * @package    qbehaviour_deferredcbm
+ * @copyright  2009 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class behaviour_type_test extends \qbehaviour_walkthrough_test_base {
+
+    /** @var qbehaviour_deferredcbm_type */
+    protected $behaviourtype;
+
+    public function setUp(): void {
+        parent::setUp();
+        $this->behaviourtype = question_engine::get_behaviour_type('deferredcbm');
+    }
+
+    public function test_is_archetypal(): void {
+        $this->assertTrue($this->behaviourtype->is_archetypal());
+    }
+
+    public function test_get_unused_display_options(): void {
+        $this->assertEquals(array('correctness', 'marks', 'specificfeedback', 'generalfeedback', 'rightanswer'),
+                $this->behaviourtype->get_unused_display_options());
+    }
+
+    public function test_can_questions_finish_during_the_attempt(): void {
+        $this->assertFalse($this->behaviourtype->can_questions_finish_during_the_attempt());
+    }
+
+    public function test_adjust_random_guess_score(): void {
+        $this->assertEquals(0, $this->behaviourtype->adjust_random_guess_score(0));
+        $this->assertEquals(1, $this->behaviourtype->adjust_random_guess_score(1));
+    }
+
+    public function test_summarise_usage_max_mark_1(): void {
+
+        // Create a usage comprising 3 true-false questions.
+        $this->quba->set_preferred_behaviour('deferredcbm');
+        $this->quba->add_question(\test_question_maker::make_question('truefalse', 'true'), 3);
+        $this->quba->add_question(\test_question_maker::make_question('truefalse', 'true'), 3);
+        $this->quba->add_question(\test_question_maker::make_question('truefalse', 'true'), 3);
+        $this->quba->start_all_questions();
+
+        // Process responses right, high certainty; right, med certainty; wrong, med certainty.
+        $this->quba->process_action(1, array('answer' => 1, '-certainty' => 3));
+        $this->quba->process_action(2, array('answer' => 1, '-certainty' => 2));
+        $this->quba->process_action(3, array('answer' => 0, '-certainty' => 2));
+        $this->quba->finish_all_questions();
+
+        // Get the summary.
+        $summarydata = $this->quba->get_summary_information(new question_display_options());
+
+        // Verify.
+        $this->assertStringContainsString(get_string('breakdownbycertainty', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement_heading']['content']);
+
+        $this->assertStringContainsString('100%',
+                $summarydata['qbehaviour_cbm_judgement3']['content']);
+        $this->assertStringContainsString(get_string('judgementok', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement3']['content']);
+
+        $this->assertStringContainsString('50%',
+                $summarydata['qbehaviour_cbm_judgement2']['content']);
+        $this->assertStringContainsString(get_string('slightlyoverconfident', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement2']['content']);
+
+        $this->assertStringContainsString(get_string('noquestions', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement1']['content']);
+    }
+
+    public function test_summarise_usage_max_mark_3(): void {
+
+        // Create a usage comprising 3 true-false questions.
+        $this->quba->set_preferred_behaviour('deferredcbm');
+        $this->quba->add_question(\test_question_maker::make_question('truefalse', 'true'), 1);
+        $this->quba->add_question(\test_question_maker::make_question('truefalse', 'true'), 1);
+        $this->quba->add_question(\test_question_maker::make_question('truefalse', 'true'), 1);
+        $this->quba->start_all_questions();
+
+        // Process responses right, high certainty; right, med certainty; wrong, med certainty.
+        $this->quba->process_action(1, array('answer' => 1, '-certainty' => 3));
+        $this->quba->process_action(2, array('answer' => 1, '-certainty' => 2));
+        $this->quba->process_action(3, array('answer' => 0, '-certainty' => 2));
+        $this->quba->finish_all_questions();
+
+        // Get the summary.
+        $summarydata = $this->quba->get_summary_information(new question_display_options());
+
+        // Verify.
+        $this->assertStringContainsString(get_string('breakdownbycertainty', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement_heading']['content']);
+
+        $this->assertStringContainsString('100%',
+                $summarydata['qbehaviour_cbm_judgement3']['content']);
+        $this->assertStringContainsString(get_string('judgementok', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement3']['content']);
+
+        $this->assertStringContainsString('50%',
+                $summarydata['qbehaviour_cbm_judgement2']['content']);
+        $this->assertStringContainsString(get_string('slightlyoverconfident', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement2']['content']);
+
+        $this->assertStringContainsString(get_string('noquestions', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement1']['content']);
+    }
+
+    public function test_calculate_bonus(): void {
+        $this->assertEqualsWithDelta(0.05,  $this->behaviourtype->calculate_bonus(1, 1 / 2), question_testcase::GRADE_DELTA);
+        $this->assertEqualsWithDelta(-0.01, $this->behaviourtype->calculate_bonus(2, 9 / 10), question_testcase::GRADE_DELTA);
+        $this->assertEqualsWithDelta(0,     $this->behaviourtype->calculate_bonus(3, 1), question_testcase::GRADE_DELTA);
+    }
+}

question/behaviour/deferredcbm/tests/question_cbm_test.php

@@ -0,0 +1,49 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace qbehaviour_deferredcbm;
+
+use question_cbm;
+
+defined('MOODLE_INTERNAL') || die();
+
+global $CFG;
+require_once(__DIR__ . '/../../../engine/lib.php');
+
+/**
+ * Unit tests for the deferred feedback with certainty base marking behaviour.
+ *
+ * @package    qbehaviour_deferredcbm
+ * @copyright  2009 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class question_cbm_test extends \basic_testcase {
+
+    public function test_adjust_fraction(): void {
+        $this->assertEqualsWithDelta( 1,   question_cbm::adjust_fraction( 1,    question_cbm::LOW), 0.0000001);
+        $this->assertEqualsWithDelta( 2,   question_cbm::adjust_fraction( 1,    question_cbm::MED), 0.0000001);
+        $this->assertEqualsWithDelta( 3,   question_cbm::adjust_fraction( 1,    question_cbm::HIGH), 0.0000001);
+        $this->assertEqualsWithDelta( 0,   question_cbm::adjust_fraction( 0,    question_cbm::LOW), 0.0000001);
+        $this->assertEqualsWithDelta(-2,   question_cbm::adjust_fraction( 0,    question_cbm::MED), 0.0000001);
+        $this->assertEqualsWithDelta(-6,   question_cbm::adjust_fraction( 0,    question_cbm::HIGH), 0.0000001);
+        $this->assertEqualsWithDelta( 0.5, question_cbm::adjust_fraction( 0.5,  question_cbm::LOW), 0.0000001);
+        $this->assertEqualsWithDelta( 1,   question_cbm::adjust_fraction( 0.5,  question_cbm::MED), 0.0000001);
+        $this->assertEqualsWithDelta( 1.5, question_cbm::adjust_fraction( 0.5,  question_cbm::HIGH), 0.0000001);
+        $this->assertEqualsWithDelta( 0,   question_cbm::adjust_fraction(-0.25, question_cbm::LOW), 0.0000001);
+        $this->assertEqualsWithDelta(-2,   question_cbm::adjust_fraction(-0.25, question_cbm::MED), 0.0000001);
+        $this->assertEqualsWithDelta(-6,   question_cbm::adjust_fraction(-0.25, question_cbm::HIGH), 0.0000001);
+    }
+}

question/behaviour/deferredcbm/tests/walkthrough_test.php

@@ -0,0 +1,277 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace qbehaviour_deferredcbm;
+
+use question_cbm;
+use question_state;
+
+defined('MOODLE_INTERNAL') || die();
+
+global $CFG;
+require_once(__DIR__ . '/../../../engine/lib.php');
+require_once(__DIR__ . '/../../../engine/tests/helpers.php');
+
+
+/**
+ * Unit tests for the deferred feedback with certainty base marking behaviour.
+ *
+ * @package    qbehaviour_deferredcbm
+ * @copyright  2009 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class walkthrough_test extends \qbehaviour_walkthrough_test_base {
+    public function test_deferred_cbm_truefalse_high_certainty(): void {
+
+        // Create a true-false question with correct answer true.
+        $tf = \test_question_maker::make_question('truefalse', 'true');
+        $this->start_attempt_at_question($tf, 'deferredcbm', 2);
+
+        // Verify.
+        $this->check_current_state(question_state::$todo);
+        $this->check_output_contains_lang_string('notyetanswered', 'question');
+        $this->check_current_mark(null);
+        $this->check_current_output(
+                $this->get_contains_question_text_expectation($tf),
+                $this->get_contains_tf_true_radio_expectation(true, false),
+                $this->get_contains_tf_false_radio_expectation(true, false),
+                $this->get_contains_cbm_radio_expectation(1, true, false),
+                $this->get_contains_cbm_radio_expectation(2, true, false),
+                $this->get_contains_cbm_radio_expectation(3, true, false),
+                $this->get_does_not_contain_feedback_expectation());
+
+        // Process the data extracted for this question.
+        $this->process_submission(array('answer' => 1, '-certainty' => 3));
+
+        // Verify.
+        $this->check_current_state(question_state::$complete);
+        $this->check_output_contains_lang_string('answersaved', 'question');
+        $this->check_current_mark(null);
+        $this->check_current_output(
+                $this->get_contains_tf_true_radio_expectation(true, true),
+                $this->get_contains_cbm_radio_expectation(3, true, true),
+                $this->get_does_not_contain_correctness_expectation(),
+                $this->get_does_not_contain_feedback_expectation());
+
+        // Process the same data again, check it does not create a new step.
+        $numsteps = $this->get_step_count();
+        $this->process_submission(array('answer' => 1, '-certainty' => 3));
+        $this->check_step_count($numsteps);
+
+        // Process different data, check it creates a new step.
+        $this->process_submission(array('answer' => 1, '-certainty' => 1));
+        $this->check_step_count($numsteps + 1);
+        $this->check_current_state(question_state::$complete);
+
+        // Change back, check it creates a new step.
+        $this->process_submission(array('answer' => 1, '-certainty' => 3));
+        $this->check_step_count($numsteps + 2);
+
+        // Finish the attempt.
+        $this->quba->finish_all_questions();
+
+        // Verify.
+        $this->check_current_state(question_state::$gradedright);
+        $this->check_current_mark(6);
+        $this->check_current_output(
+                $this->get_contains_tf_true_radio_expectation(false, true),
+                $this->get_contains_cbm_radio_expectation(3, false, true),
+                $this->get_contains_correct_expectation());
+
+        // Process a manual comment.
+        $this->manual_grade('Not good enough!', 5, FORMAT_HTML);
+
+        // Verify.
+        $this->check_current_state(question_state::$mangrright);
+        $this->check_current_mark(5);
+        $this->check_current_output(new \question_pattern_expectation('/' .
+                preg_quote('Not good enough!', '/') . '/'));
+
+        // Now change the correct answer to the question, and regrade.
+        $tf->rightanswer = false;
+        $this->quba->regrade_all_questions();
+
+        // Verify.
+        $this->check_current_state(question_state::$mangrright);
+        $this->check_current_mark(5);
+        $autogradedstep = $this->get_step($this->get_step_count() - 2);
+        $this->assertEqualsWithDelta(-6, $autogradedstep->get_fraction(), 0.0000001);
+    }
+
+    public function test_deferred_cbm_truefalse_low_certainty(): void {
+
+        // Create a true-false question with correct answer true.
+        $tf = \test_question_maker::make_question('truefalse', 'true');
+        $this->start_attempt_at_question($tf, 'deferredcbm', 2);
+
+        // Verify.
+        $this->check_current_state(question_state::$todo);
+        $this->check_output_contains_lang_string('notyetanswered', 'question');
+        $this->check_current_mark(null);
+        $this->check_current_output(
+                $this->get_does_not_contain_correctness_expectation(),
+                $this->get_contains_cbm_radio_expectation(1, true, false),
+                $this->get_does_not_contain_feedback_expectation());
+
+        // Submit ansewer with low certainty.
+        $this->process_submission(array('answer' => 1, '-certainty' => 1));
+
+        // Verify.
+        $this->check_current_state(question_state::$complete);
+        $this->check_output_contains_lang_string('answersaved', 'question');
+        $this->check_current_mark(null);
+        $this->check_current_output($this->get_does_not_contain_correctness_expectation(),
+                $this->get_contains_cbm_radio_expectation(1, true, true),
+                $this->get_does_not_contain_feedback_expectation());
+
+        // Finish the attempt.
+        $this->quba->finish_all_questions();
+
+        // Verify.
+        $this->check_current_state(question_state::$gradedright);
+        $this->check_current_mark(2);
+        $this->check_current_output($this->get_contains_correct_expectation(),
+                $this->get_contains_cbm_radio_expectation(1, false, true));
+        $this->assertEquals(get_string('true', 'qtype_truefalse') . ' [' .
+                question_cbm::get_short_string(question_cbm::LOW) . ']',
+                $this->quba->get_response_summary($this->slot));
+    }
+
+    public function test_deferred_cbm_truefalse_default_certainty(): void {
+
+        // Create a true-false question with correct answer true.
+        $tf = \test_question_maker::make_question('truefalse', 'true');
+        $this->start_attempt_at_question($tf, 'deferredcbm', 2);
+
+        // Verify.
+        $this->check_current_state(question_state::$todo);
+        $this->check_output_contains_lang_string('notyetanswered', 'question');
+        $this->check_current_mark(null);
+        $this->check_current_output(
+                $this->get_does_not_contain_correctness_expectation(),
+                $this->get_contains_cbm_radio_expectation(1, true, false),
+                $this->get_does_not_contain_feedback_expectation());
+
+        // Submit ansewer with low certainty and finish the attempt.
+        $this->process_submission(array('answer' => 1));
+        $this->quba->finish_all_questions();
+
+        // Verify.
+        $qa = $this->quba->get_question_attempt($this->slot);
+        $this->check_current_state(question_state::$gradedright);
+        $this->check_current_mark(2);
+        $this->check_current_output($this->get_contains_correct_expectation(),
+                $this->get_contains_cbm_radio_expectation(1, false, false),
+                new \question_pattern_expectation('/' . preg_quote(
+                        get_string('assumingcertainty', 'qbehaviour_deferredcbm',
+                        question_cbm::get_string(
+                            $qa->get_last_behaviour_var('_assumedcertainty'))), '/') . '/'));
+        $this->assertEquals(get_string('true', 'qtype_truefalse'),
+                $this->quba->get_response_summary($this->slot));
+    }
+
+    public function test_deferredcbm_resume_multichoice_single(): void {
+
+        // Create a multiple-choice question.
+        $mc = \test_question_maker::make_a_multichoice_single_question();
+
+        // Attempt it getting it wrong.
+        $this->start_attempt_at_question($mc, 'deferredcbm', 1);
+        $rightindex = $this->get_mc_right_answer_index($mc);
+        $wrongindex = ($rightindex + 1) % 3;
+        $this->process_submission(array('answer' => $wrongindex, '-certainty' => 2));
+        $this->quba->finish_all_questions();
+
+        // Verify.
+        $this->check_current_state(question_state::$gradedwrong);
+        $this->check_current_mark(-2);
+        $this->check_current_output(
+                $this->get_contains_mc_radio_expectation($wrongindex, false, true),
+                $this->get_contains_cbm_radio_expectation(2, false, true),
+                $this->get_contains_incorrect_expectation());
+        $this->assertEquals('A [' . question_cbm::get_short_string(question_cbm::HIGH) . ']',
+                $this->quba->get_right_answer_summary($this->slot));
+        $this->assertMatchesRegularExpression('/' . preg_quote($mc->questiontext, '/') . '/',
+                $this->quba->get_question_summary($this->slot));
+        $this->assertMatchesRegularExpression(
+            '/(B|C) \[' . preg_quote(question_cbm::get_short_string(question_cbm::MED), '/') . '\]/',
+            $this->quba->get_response_summary($this->slot));
+
+        // Save the old attempt.
+        $oldqa = $this->quba->get_question_attempt($this->slot);
+
+        // Reinitialise.
+        $this->setUp();
+        $this->quba->set_preferred_behaviour('deferredcbm');
+        $this->slot = $this->quba->add_question($mc, 1);
+        $this->quba->start_question_based_on($this->slot, $oldqa);
+
+        // Verify.
+        $this->check_current_state(question_state::$complete);
+        $this->check_output_contains_lang_string('notchanged', 'question');
+        $this->check_current_mark(null);
+        $this->check_current_output(
+                $this->get_contains_mc_radio_expectation($wrongindex, true, true),
+                $this->get_contains_cbm_radio_expectation(2, true, true),
+                $this->get_does_not_contain_feedback_expectation(),
+                $this->get_does_not_contain_correctness_expectation());
+        $this->assertEquals('A [' . question_cbm::get_short_string(question_cbm::HIGH) . ']',
+                $this->quba->get_right_answer_summary($this->slot));
+        $this->assertMatchesRegularExpression('/' . preg_quote($mc->questiontext, '/') . '/',
+                $this->quba->get_question_summary($this->slot));
+        $this->assertNull($this->quba->get_response_summary($this->slot));
+
+        // Now get it right.
+        $this->process_submission(array('answer' => $rightindex, '-certainty' => 3));
+        $this->quba->finish_all_questions();
+
+        // Verify.
+        $this->check_current_state(question_state::$gradedright);
+        $this->check_current_mark(3);
+        $this->check_current_output(
+                $this->get_contains_mc_radio_expectation($rightindex, false, true),
+                $this->get_contains_cbm_radio_expectation(question_cbm::HIGH, false, true),
+                $this->get_contains_correct_expectation());
+        $this->assertMatchesRegularExpression(
+            '/(A) \[' . preg_quote(question_cbm::get_short_string(question_cbm::HIGH), '/') . '\]/',
+            $this->quba->get_response_summary($this->slot));
+    }
+
+    public function test_deferred_cbm_truefalse_no_certainty_feedback_when_not_answered(): void {
+
+        // Create a true-false question with correct answer true.
+        $tf = \test_question_maker::make_question('truefalse', 'true');
+        $this->start_attempt_at_question($tf, 'deferredcbm', 2);
+
+        // Verify.
+        $this->check_current_state(question_state::$todo);
+        $this->check_output_contains_lang_string('notyetanswered', 'question');
+        $this->check_current_mark(null);
+        $this->check_current_output(
+                $this->get_does_not_contain_correctness_expectation(),
+                $this->get_contains_cbm_radio_expectation(1, true, false),
+                $this->get_does_not_contain_feedback_expectation());
+
+        // Finish without answering.
+        $this->quba->finish_all_questions();
+
+        // Verify.
+        $this->check_current_state(question_state::$gaveup);
+        $this->check_current_mark(null);
+        $this->check_current_output(
+                new \question_no_pattern_expectation('/class=\"im-feedback/'));
+    }
+}

question/behaviour/deferredcbm/version.php

@@ -0,0 +1,36 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Version information for the calculated question type.
+ *
+ * @package    qbehaviour
+ * @subpackage deferredcbm
+ * @copyright  2011 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+$plugin->component = 'qbehaviour_deferredcbm';
+$plugin->version   = 2024042200;
+
+$plugin->requires  = 2024041600;
+$plugin->dependencies = [
+    'qbehaviour_deferredfeedback' => 2024041600,
+];
+
+$plugin->maturity  = MATURITY_STABLE;