first commit

question/engine/upgrade/behaviourconverters.php

@@ -0,0 +1,565 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * This file contains classes for handling the different question behaviours
+ * during upgrade.
+ *
+ * @package    moodlecore
+ * @subpackage questionengine
+ * @copyright  2010 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+
+defined('MOODLE_INTERNAL') || die();
+
+
+/**
+ * Base class for managing the upgrade of a question using a particular behaviour.
+ *
+ * This class takes as input:
+ * 1. Various backgroud data like $quiz, $attempt and $question.
+ * 2. The data about the question session to upgrade $qsession and $qstates.
+ * Working through that data, it builds up
+ * 3. The equivalent new data $qa. This has roughly the same data as a
+ * question_attempt object belonging to the new question engine would have, but
+ * $this->qa is built up from stdClass objects.
+ *
+ * @copyright  2010 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+abstract class question_behaviour_attempt_updater {
+    /** @var question_qtype_attempt_updater */
+    protected $qtypeupdater;
+    /** @var question_engine_assumption_logger */
+    protected $logger;
+    /** @var question_engine_attempt_upgrader */
+    protected $qeupdater;
+
+    /**
+     * @var object this is the data for the upgraded questions attempt that
+     * we are building.
+     */
+    public $qa;
+
+    /** @var object the quiz settings. */
+    protected $quiz;
+    /** @var object the quiz attempt data. */
+    protected $attempt;
+    /** @var object the question definition data. */
+    protected $question;
+    /** @var object the question session to be upgraded. */
+    protected $qsession;
+    /** @var array the question states for the session to be upgraded. */
+    protected $qstates;
+    /** @var stdClass */
+    protected $startstate;
+
+    /**
+     * @var int counts the question_steps as they are converted to
+     * question_attempt_steps.
+     */
+    protected $sequencenumber;
+    /** @var object pointer to the state that has already finished this attempt. */
+    protected $finishstate;
+
+    public function __construct($quiz, $attempt, $question, $qsession, $qstates, $logger, $qeupdater) {
+        $this->quiz = $quiz;
+        $this->attempt = $attempt;
+        $this->question = $question;
+        $this->qsession = $qsession;
+        $this->qstates = $qstates;
+        $this->logger = $logger;
+        $this->qeupdater = $qeupdater;
+    }
+
+    public function discard() {
+        // Help the garbage collector, which seems to be struggling.
+        $this->quiz = null;
+        $this->attempt = null;
+        $this->question = null;
+        $this->qsession = null;
+        $this->qstates = null;
+        $this->qa = null;
+        $this->qtypeupdater->discard();
+        $this->qtypeupdater = null;
+        $this->logger = null;
+        $this->qeupdater = null;
+    }
+
+    abstract protected function behaviour_name();
+
+    public function get_converted_qa() {
+        $this->initialise_qa();
+        $this->convert_steps();
+        return $this->qa;
+    }
+
+    protected function create_missing_first_step() {
+        $step = new stdClass();
+        $step->state = 'todo';
+        $step->data = array();
+        $step->fraction = null;
+        $step->timecreated = $this->attempt->timestart ? $this->attempt->timestart : time();
+        $step->userid = $this->attempt->userid;
+        $this->qtypeupdater->supply_missing_first_step_data($step->data);
+        return $step;
+    }
+
+    public function supply_missing_qa() {
+        $this->initialise_qa();
+        $this->qa->timemodified = $this->attempt->timestart;
+        $this->sequencenumber = 0;
+        $this->add_step($this->create_missing_first_step());
+        return $this->qa;
+    }
+
+    protected function initialise_qa() {
+        $this->qtypeupdater = $this->make_qtype_updater();
+
+        $qa = new stdClass();
+        $qa->questionid = $this->question->id;
+        $qa->variant = 1;
+        $qa->behaviour = $this->behaviour_name();
+        $qa->questionsummary = $this->qtypeupdater->question_summary($this->question);
+        $qa->rightanswer = $this->qtypeupdater->right_answer($this->question);
+        $qa->maxmark = $this->question->maxmark;
+        $qa->minfraction = 0;
+        $qa->maxfraction = 1;
+        $qa->flagged = 0;
+        $qa->responsesummary = '';
+        $qa->timemodified = 0;
+        $qa->steps = array();
+
+        $this->qa = $qa;
+    }
+
+    protected function convert_steps() {
+        $this->finishstate = null;
+        $this->startstate = null;
+        $this->sequencenumber = 0;
+        foreach ($this->qstates as $state) {
+            $this->process_state($state);
+        }
+        $this->finish_up();
+    }
+
+    protected function process_state($state) {
+        $step = $this->make_step($state);
+        $method = 'process' . $state->event;
+        $this->$method($step, $state);
+    }
+
+    protected function finish_up() {
+    }
+
+    protected function add_step($step) {
+        $step->sequencenumber = $this->sequencenumber;
+        $this->qa->steps[] = $step;
+        $this->sequencenumber++;
+    }
+
+    protected function discard_last_state() {
+        array_pop($this->qa->steps);
+        $this->sequencenumber--;
+    }
+
+    protected function unexpected_event($state) {
+        throw new coding_exception("Unexpected event {$state->event} in state {$state->id} in question session {$this->qsession->id}.");
+    }
+
+    protected function process0($step, $state) {
+        if ($this->startstate) {
+            if ($state->answer == reset($this->qstates)->answer) {
+                return;
+            } else if ($this->quiz->attemptonlast && $this->sequencenumber == 1) {
+                // There was a bug in attemptonlast in the past, which meant that
+                // it created two inconsistent open states, with the second taking
+                // priority. Simulate that be discarding the first open state, then
+                // continuing.
+                $this->logger->log_assumption("Ignoring bogus state in attempt at question {$state->question}");
+                $this->sequencenumber = 0;
+                $this->qa->steps = array();
+            } else if ($this->qtypeupdater->is_blank_answer($state)) {
+                $this->logger->log_assumption("Ignoring second start state with blank answer in attempt at question {$state->question}");
+                return;
+            } else {
+                throw new coding_exception("Two inconsistent open states for question session {$this->qsession->id}.");
+            }
+        }
+        $step->state = 'todo';
+        $this->startstate = $state;
+        $this->add_step($step);
+    }
+
+    protected function process1($step, $state) {
+        $this->unexpected_event($state);
+    }
+
+    protected function process2($step, $state) {
+        if ($this->qtypeupdater->was_answered($state)) {
+            $step->state = 'complete';
+        } else {
+            $step->state = 'todo';
+        }
+        $this->add_step($step);
+    }
+
+    protected function process3($step, $state) {
+        return $this->process6($step, $state);
+    }
+
+    protected function process4($step, $state) {
+        $this->unexpected_event($state);
+    }
+
+    protected function process5($step, $state) {
+        $this->unexpected_event($state);
+    }
+
+    abstract protected function process6($step, $state);
+    abstract protected function process7($step, $state);
+
+    protected function process8($step, $state) {
+        return $this->process6($step, $state);
+    }
+
+    protected function process9($step, $state) {
+        if (!$this->finishstate) {
+            $submitstate = clone($state);
+            $submitstate->event = 8;
+            $submitstate->grade = 0;
+            $this->process_state($submitstate);
+        }
+
+        $step->data['-comment'] = $this->qsession->manualcomment;
+        if ($this->question->maxmark > 0) {
+            $step->fraction = $state->grade / $this->question->maxmark;
+            $step->state = $this->manual_graded_state_for_fraction($step->fraction);
+            $step->data['-mark'] = $state->grade;
+            $step->data['-maxmark'] = $this->question->maxmark;
+        } else {
+            $step->state = 'manfinished';
+        }
+        unset($step->data['answer']);
+        $step->userid = null;
+        $this->add_step($step);
+    }
+
+    /**
+     * @param object $question a question definition
+     * @return qtype_updater
+     */
+    protected function make_qtype_updater() {
+        global $CFG;
+
+        if ($this->question->qtype == 'deleted') {
+            return new question_deleted_question_attempt_updater(
+                    $this, $this->question, $this->logger, $this->qeupdater);
+        }
+
+        $path = $CFG->dirroot . '/question/type/' . $this->question->qtype . '/db/upgradelib.php';
+        if (!is_readable($path)) {
+            throw new coding_exception("Question type {$this->question->qtype}
+                    is missing important code (the file {$path})
+                    required to run the upgrade to the new question engine.");
+        }
+        include_once($path);
+        $class = 'qtype_' . $this->question->qtype . '_qe2_attempt_updater';
+        if (!class_exists($class)) {
+            throw new coding_exception("Question type {$this->question->qtype}
+                    is missing important code (the class {$class})
+                    required to run the upgrade to the new question engine.");
+        }
+        return new $class($this, $this->question, $this->logger, $this->qeupdater);
+    }
+
+    public function to_text($html) {
+        return trim(html_to_text($html, 0, false));
+    }
+
+    protected function graded_state_for_fraction($fraction) {
+        if ($fraction < 0.000001) {
+            return 'gradedwrong';
+        } else if ($fraction > 0.999999) {
+            return 'gradedright';
+        } else {
+            return 'gradedpartial';
+        }
+    }
+
+    protected function manual_graded_state_for_fraction($fraction) {
+        if ($fraction < 0.000001) {
+            return 'mangrwrong';
+        } else if ($fraction > 0.999999) {
+            return 'mangrright';
+        } else {
+            return 'mangrpartial';
+        }
+    }
+
+    protected function make_step($state){
+        $step = new stdClass();
+        $step->data = array();
+
+        if ($state->event == 0 || $this->sequencenumber == 0) {
+            $this->qtypeupdater->set_first_step_data_elements($state, $step->data);
+        } else {
+            $this->qtypeupdater->set_data_elements_for_step($state, $step->data);
+        }
+
+        $step->fraction = null;
+        $step->timecreated = $state->timestamp ? $state->timestamp : time();
+        $step->userid = $this->attempt->userid;
+
+        $summary = $this->qtypeupdater->response_summary($state);
+        if (!is_null($summary)) {
+            $this->qa->responsesummary = $summary;
+        }
+        $this->qa->timemodified = max($this->qa->timemodified, $state->timestamp);
+
+        return $step;
+    }
+}
+
+
+class qbehaviour_deferredfeedback_converter extends question_behaviour_attempt_updater {
+
+    protected function behaviour_name() {
+        return 'deferredfeedback';
+    }
+
+    protected function process6($step, $state) {
+        if (!$this->startstate) {
+            $this->logger->log_assumption("Ignoring bogus submit before open in attempt at question {$state->question}");
+            // WTF, but this has happened a few times in our DB. It seems it is safe to ignore.
+            return;
+        }
+
+        if ($this->finishstate) {
+            if ($this->finishstate->answer != $state->answer ||
+                    $this->finishstate->grade != $state->grade ||
+                    $this->finishstate->raw_grade != $state->raw_grade ||
+                    $this->finishstate->penalty != $state->penalty) {
+                $this->logger->log_assumption("Two inconsistent finish states found for question session {$this->qsession->id} in attempt at question {$state->question} keeping the later one.");
+                $this->discard_last_state();
+            } else {
+                $this->logger->log_assumption("Ignoring extra finish states in attempt at question {$state->question}");
+                return;
+            }
+        }
+
+        if ($this->question->maxmark > 0) {
+            $step->fraction = $state->grade / $this->question->maxmark;
+            $step->state = $this->graded_state_for_fraction($step->fraction);
+        } else {
+            $step->state = 'finished';
+        }
+        $step->data['-finish'] = '1';
+        $this->finishstate = $state;
+        $this->add_step($step);
+    }
+
+    protected function process7($step, $state) {
+        $this->unexpected_event($state);
+    }
+}
+
+
+class qbehaviour_manualgraded_converter extends question_behaviour_attempt_updater {
+    protected function behaviour_name() {
+        return 'manualgraded';
+    }
+
+    protected function process6($step, $state) {
+        $step->state = 'needsgrading';
+        if (!$this->finishstate) {
+            $step->data['-finish'] = '1';
+            $this->finishstate = $state;
+        }
+        $this->add_step($step);
+    }
+
+    protected function process7($step, $state) {
+        return $this->process2($step, $state);
+    }
+}
+
+
+class qbehaviour_informationitem_converter extends question_behaviour_attempt_updater {
+
+    protected function behaviour_name() {
+        return 'informationitem';
+    }
+
+    protected function process0($step, $state) {
+        if ($this->startstate) {
+            return;
+        }
+        $step->state = 'todo';
+        $this->startstate = $state;
+        $this->add_step($step);
+    }
+
+    protected function process2($step, $state) {
+        $this->unexpected_event($state);
+    }
+
+    protected function process3($step, $state) {
+        $this->unexpected_event($state);
+    }
+
+    protected function process6($step, $state) {
+        if ($this->finishstate) {
+            return;
+        }
+
+        $step->state = 'finished';
+        $step->data['-finish'] = '1';
+        $this->finishstate = $state;
+        $this->add_step($step);
+    }
+
+    protected function process7($step, $state) {
+        return $this->process6($step, $state);
+    }
+
+    protected function process8($step, $state) {
+        return $this->process6($step, $state);
+    }
+}
+
+
+class qbehaviour_adaptive_converter extends question_behaviour_attempt_updater {
+    protected $try;
+    protected $laststepwasatry = false;
+    protected $finished = false;
+    protected $bestrawgrade = 0;
+
+    protected function behaviour_name() {
+        return 'adaptive';
+    }
+
+    protected function finish_up() {
+        parent::finish_up();
+        if ($this->finishstate || !$this->attempt->timefinish) {
+            return;
+        }
+
+        $state = end($this->qstates);
+        $step = $this->make_step($state);
+        $this->process6($step, $state);
+    }
+
+    protected function process0($step, $state) {
+        $this->try = 1;
+        $this->laststepwasatry = false;
+        parent::process0($step, $state);
+    }
+
+    protected function process2($step, $state) {
+        if ($this->finishstate) {
+            $this->logger->log_assumption("Ignoring bogus save after submit in an " .
+                    "adaptive attempt at question {$state->question} " .
+                    "(question session {$this->qsession->id})");
+            return;
+        }
+
+        if ($this->question->maxmark > 0) {
+            $step->fraction = $state->grade / $this->question->maxmark;
+        }
+
+        $this->laststepwasatry = false;
+        parent::process2($step, $state);
+    }
+
+    protected function process3($step, $state) {
+        if ($this->question->maxmark > 0) {
+            $step->fraction = $state->grade / $this->question->maxmark;
+            if ($this->graded_state_for_fraction($step->fraction) == 'gradedright') {
+                $step->state = 'complete';
+            } else {
+                $step->state = 'todo';
+            }
+        } else {
+            $step->state = 'complete';
+        }
+
+        $this->bestrawgrade = max($state->raw_grade, $this->bestrawgrade);
+
+        $step->data['-_try'] = $this->try;
+        $this->try += 1;
+        $this->laststepwasatry = true;
+        if ($this->question->maxmark > 0) {
+            $step->data['-_rawfraction'] = $state->raw_grade / $this->question->maxmark;
+        } else {
+            $step->data['-_rawfraction'] = 0;
+        }
+        $step->data['-submit'] = 1;
+
+        $this->add_step($step);
+    }
+
+    protected function process6($step, $state) {
+        if ($this->finishstate) {
+            if (!$this->qtypeupdater->compare_answers($this->finishstate->answer, $state->answer) ||
+                    $this->finishstate->grade != $state->grade ||
+                    $this->finishstate->raw_grade != $state->raw_grade ||
+                    $this->finishstate->penalty != $state->penalty) {
+                throw new coding_exception("Two inconsistent finish states found for question session {$this->qsession->id}.");
+            } else {
+                $this->logger->log_assumption("Ignoring extra finish states in attempt at question {$state->question}");
+                return;
+            }
+        }
+
+        $this->bestrawgrade = max($state->raw_grade, $this->bestrawgrade);
+
+        if ($this->question->maxmark > 0) {
+            $step->fraction = $state->grade / $this->question->maxmark;
+            $step->state = $this->graded_state_for_fraction(
+                    $this->bestrawgrade / $this->question->maxmark);
+        } else {
+            $step->state = 'finished';
+        }
+
+        $step->data['-finish'] = 1;
+        if ($this->laststepwasatry) {
+            $this->try -= 1;
+        }
+        $step->data['-_try'] = $this->try;
+        if ($this->question->maxmark > 0) {
+            $step->data['-_rawfraction'] = $state->raw_grade / $this->question->maxmark;
+        } else {
+            $step->data['-_rawfraction'] = 0;
+        }
+
+        $this->finishstate = $state;
+        $this->add_step($step);
+    }
+
+    protected function process7($step, $state) {
+        $this->unexpected_event($state);
+    }
+}
+
+
+class qbehaviour_adaptivenopenalty_converter extends qbehaviour_adaptive_converter {
+    protected function behaviour_name() {
+        return 'adaptivenopenalty';
+    }
+}

question/engine/upgrade/logger.php

@@ -0,0 +1,95 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Code that deals with logging stuff during the question engine upgrade.
+ *
+ * @package    moodlecore
+ * @subpackage questionengine
+ * @copyright  2010 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+
+defined('MOODLE_INTERNAL') || die();
+
+
+/**
+ * This class serves to record all the assumptions that the code had to make
+ * during the question engine database database upgrade, to facilitate reviewing
+ * them.
+ *
+ * @copyright  2010 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class question_engine_assumption_logger {
+    protected $handle;
+    protected $attemptid;
+
+    public function __construct() {
+        global $CFG;
+        make_upload_directory('upgradelogs');
+        $date = date('Ymd-His');
+        $this->handle = fopen($CFG->dataroot . '/upgradelogs/qe_' .
+                $date . '.html', 'a');
+        fwrite($this->handle, '<html><head><title>Question engine upgrade assumptions ' .
+                $date . '</title></head><body><h2>Question engine upgrade assumptions ' .
+                $date . "</h2>\n\n");
+    }
+
+    public function set_current_attempt_id($id) {
+        $this->attemptid = $id;
+    }
+
+    public function log_assumption($description, $quizattemptid = null) {
+        global $CFG;
+        $message = '<p>' . $description;
+        if (!$quizattemptid) {
+            $quizattemptid = $this->attemptid;
+        }
+        if ($quizattemptid) {
+            $message .= ' (<a href="' . $CFG->wwwroot . '/mod/quiz/review.php?attempt=' .
+                    $quizattemptid . '">Review this attempt</a>)';
+        }
+        $message .= "</p>\n";
+        fwrite($this->handle, $message);
+    }
+
+    public function __destruct() {
+        fwrite($this->handle, '</body></html>');
+        fclose($this->handle);
+    }
+}
+
+
+/**
+ * Subclass of question_engine_assumption_logger that does nothing, for testing.
+ *
+ * @copyright  2009 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class dummy_question_engine_assumption_logger extends question_engine_assumption_logger {
+    protected $attemptid;
+
+    public function __construct() {
+    }
+
+    public function log_assumption($description, $quizattemptid = null) {
+    }
+
+    public function __destruct() {
+    }
+}

question/engine/upgrade/tests/helper.php

@@ -0,0 +1,141 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * This file contains test helper code for testing the upgrade to the new
+ * question engine. The actual tests are organised by question type in files
+ * like question/type/truefalse/tests/upgradelibnewqe_test.php.
+ *
+ * @package    moodlecore
+ * @subpackage questionengine
+ * @copyright  2009 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+
+defined('MOODLE_INTERNAL') || die();
+
+global $CFG;
+require_once(__DIR__ . '/../upgradelib.php');
+
+
+/**
+ * Subclass of question_engine_attempt_upgrader to help with testing.
+ *
+ * @copyright  2009 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class test_question_engine_attempt_upgrader extends question_engine_attempt_upgrader {
+    public function prevent_timeout() {
+    }
+
+    public function __construct($loader, $logger) {
+        $this->questionloader = $loader;
+        $this->logger = $logger;
+    }
+}
+
+
+/**
+ * Subclass of question_engine_upgrade_question_loader for unit testing.
+ *
+ * @copyright  2009 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class test_question_engine_upgrade_question_loader extends question_engine_upgrade_question_loader {
+    public function put_question_in_cache($question) {
+        $this->cache[$question->id] = $question;
+    }
+
+    public function load_question($questionid, $quizid) {
+        global $CFG;
+
+        if (isset($this->cache[$questionid])) {
+            return $this->cache[$questionid];
+        }
+
+        return null;
+    }
+
+    public function put_dataset_in_cache($questionid, $selecteditem, $dataset) {
+        $this->datasetcache[$questionid][$selecteditem] = $dataset;
+    }
+
+    public function load_dataset($questionid, $selecteditem) {
+        global $DB;
+
+        if (isset($this->datasetcache[$questionid][$selecteditem])) {
+            return $this->datasetcache[$questionid][$selecteditem];
+        }
+
+        throw new coding_exception('Test dataset not loaded.');
+    }
+}
+
+
+/**
+ * Base class for tests that thest the upgrade of one particular attempt and
+ * one question.
+ *
+ * @copyright  2009 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+abstract class question_attempt_upgrader_test_base extends advanced_testcase {
+    protected $updater;
+    protected $loader;
+
+    protected function setUp(): void {
+        parent::setUp();
+        $logger = new dummy_question_engine_assumption_logger();
+        $this->loader = new test_question_engine_upgrade_question_loader($logger);
+        $this->updater = new test_question_engine_attempt_upgrader($this->loader, $logger);
+    }
+
+    protected function tearDown(): void {
+        $this->updater = null;
+        parent::tearDown();
+    }
+
+    /**
+     * Clear text, bringing independence of html2text results
+     *
+     * Some tests performing text comparisons of converted text are too much
+     * dependent of the behavior of the html2text library. This function is
+     * aimed to reduce such dependencies that should not affect the results
+     * of these question attempt upgrade tests.
+     */
+    protected function clear_html2text_dependencies($qa) {
+        // Cleaning all whitespace should be enough to ignore any html2text dependency
+        if (property_exists($qa, 'responsesummary')) {
+            $qa->responsesummary = preg_replace('/\s/', '', $qa->responsesummary);
+        }
+        if (property_exists($qa, 'questionsummary')) {
+            $qa->questionsummary = preg_replace('/\s/', '', $qa->questionsummary);
+        }
+    }
+
+    /**
+     * Compare two qas, ignoring inessential differences.
+     * @param object $expectedqa the expected qa.
+     * @param object $qa the actual qa.
+     */
+    protected function compare_qas($expectedqa, $qa) {
+        $this->clear_html2text_dependencies($expectedqa);
+        $this->clear_html2text_dependencies($qa);
+
+        $this->assertEquals($expectedqa, $qa);
+    }
+}

question/engine/upgrade/upgradelib.php

@@ -0,0 +1,484 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * This file contains the code required to upgrade all the attempt data from
+ * old versions of Moodle into the tables used by the new question engine.
+ *
+ * @package    moodlecore
+ * @subpackage questionengine
+ * @copyright  2010 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+
+defined('MOODLE_INTERNAL') || die();
+
+global $CFG;
+require_once($CFG->dirroot . '/question/engine/bank.php');
+require_once($CFG->dirroot . '/question/engine/upgrade/logger.php');
+require_once($CFG->dirroot . '/question/engine/upgrade/behaviourconverters.php');
+
+
+/**
+ * This class manages upgrading all the question attempts from the old database
+ * structure to the new question engine.
+ *
+ * @copyright  2010 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class question_engine_attempt_upgrader {
+    /** @var question_engine_upgrade_question_loader */
+    protected $questionloader;
+    /** @var question_engine_assumption_logger */
+    protected $logger;
+    /** @var stdClass */
+    protected $qsession;
+
+    public function save_usage($preferredbehaviour, $attempt, $qas, $quizlayout) {
+        global $OUTPUT;
+
+        $missing = array();
+
+        $layout = explode(',', $attempt->layout);
+        $questionkeys = array_combine(array_values($layout), array_keys($layout));
+
+        $this->set_quba_preferred_behaviour($attempt->uniqueid, $preferredbehaviour);
+
+        $i = 0;
+        foreach (explode(',', $quizlayout) as $questionid) {
+            if ($questionid == 0) {
+                continue;
+            }
+            $i++;
+
+            if (!array_key_exists($questionid, $qas)) {
+                $missing[] = $questionid;
+                $layout[$questionkeys[$questionid]] = $questionid;
+                continue;
+            }
+
+            $qa = $qas[$questionid];
+            $qa->questionusageid = $attempt->uniqueid;
+            $qa->slot = $i;
+            if (core_text::strlen($qa->questionsummary) > question_bank::MAX_SUMMARY_LENGTH) {
+                // It seems some people write very long quesions! MDL-30760
+                $qa->questionsummary = core_text::substr($qa->questionsummary,
+                        0, question_bank::MAX_SUMMARY_LENGTH - 3) . '...';
+            }
+            $this->insert_record('question_attempts', $qa);
+            $layout[$questionkeys[$questionid]] = $qa->slot;
+
+            foreach ($qa->steps as $step) {
+                $step->questionattemptid = $qa->id;
+                $this->insert_record('question_attempt_steps', $step);
+
+                foreach ($step->data as $name => $value) {
+                    $datum = new stdClass();
+                    $datum->attemptstepid = $step->id;
+                    $datum->name = $name;
+                    $datum->value = $value;
+                    $this->insert_record('question_attempt_step_data', $datum, false);
+                }
+            }
+        }
+
+        $this->set_quiz_attempt_layout($attempt->uniqueid, implode(',', $layout));
+
+        if ($missing) {
+            $message = "Question sessions for questions " .
+                    implode(', ', $missing) .
+                    " were missing when upgrading question usage {$attempt->uniqueid}.";
+            echo $OUTPUT->notification($message);
+        }
+    }
+
+    protected function set_quba_preferred_behaviour($qubaid, $preferredbehaviour) {
+        global $DB;
+        $DB->set_field('question_usages', 'preferredbehaviour', $preferredbehaviour,
+                array('id' => $qubaid));
+    }
+
+    protected function set_quiz_attempt_layout($qubaid, $layout) {
+        global $DB;
+        $DB->set_field('quiz_attempts', 'layout', $layout, array('uniqueid' => $qubaid));
+    }
+
+    protected function delete_quiz_attempt($qubaid) {
+        global $DB;
+        $DB->delete_records('quiz_attempts', array('uniqueid' => $qubaid));
+        $DB->delete_records('question_attempts', array('id' => $qubaid));
+    }
+
+    protected function insert_record($table, $record, $saveid = true) {
+        global $DB;
+        $newid = $DB->insert_record($table, $record, $saveid);
+        if ($saveid) {
+            $record->id = $newid;
+        }
+        return $newid;
+    }
+
+    public function load_question($questionid, $quizid = null) {
+        return $this->questionloader->get_question($questionid, $quizid);
+    }
+
+    public function load_dataset($questionid, $selecteditem) {
+        return $this->questionloader->load_dataset($questionid, $selecteditem);
+    }
+
+    public function get_next_question_session($attempt, moodle_recordset $questionsessionsrs) {
+        if (!$questionsessionsrs->valid()) {
+            return false;
+        }
+
+        $qsession = $questionsessionsrs->current();
+        if ($qsession->attemptid != $attempt->uniqueid) {
+            // No more question sessions belonging to this attempt.
+            return false;
+        }
+
+        // Session found, move the pointer in the RS and return the record.
+        $questionsessionsrs->next();
+        return $qsession;
+    }
+
+    public function get_question_states($attempt, $question, moodle_recordset $questionsstatesrs) {
+        $qstates = array();
+
+        while ($questionsstatesrs->valid()) {
+            $state = $questionsstatesrs->current();
+            if ($state->attempt != $attempt->uniqueid ||
+                    $state->question != $question->id) {
+                // We have found all the states for this attempt. Stop.
+                break;
+            }
+
+            // Add the new state to the array, and advance.
+            $qstates[] = $state;
+            $questionsstatesrs->next();
+        }
+
+        return $qstates;
+    }
+
+    protected function get_converter_class_name($question, $quiz, $qsessionid) {
+        global $DB;
+        if ($question->qtype == 'deleted') {
+            $where = '(question = :questionid OR '.$DB->sql_like('answer', ':randomid').') AND event = 7';
+            $params = array('questionid'=>$question->id, 'randomid'=>"random{$question->id}-%");
+            if ($DB->record_exists_select('question_states', $where, $params)) {
+                $this->logger->log_assumption("Assuming that deleted question {$question->id} was manually graded.");
+                return 'qbehaviour_manualgraded_converter';
+            }
+        }
+        $qtype = question_bank::get_qtype($question->qtype, false);
+        if ($qtype->is_manual_graded()) {
+            return 'qbehaviour_manualgraded_converter';
+        } else if ($question->qtype == 'description') {
+            return 'qbehaviour_informationitem_converter';
+        } else if ($quiz->preferredbehaviour == 'deferredfeedback') {
+            return 'qbehaviour_deferredfeedback_converter';
+        } else if ($quiz->preferredbehaviour == 'adaptive') {
+            return 'qbehaviour_adaptive_converter';
+        } else if ($quiz->preferredbehaviour == 'adaptivenopenalty') {
+            return 'qbehaviour_adaptivenopenalty_converter';
+        } else {
+            throw new coding_exception("Question session {$qsessionid}
+                    has an unexpected preferred behaviour {$quiz->preferredbehaviour}.");
+        }
+    }
+
+    public function supply_missing_question_attempt($quiz, $attempt, $question) {
+        if ($question->qtype == 'random') {
+            throw new coding_exception("Cannot supply a missing qsession for question
+                    {$question->id} in attempt {$attempt->id}.");
+        }
+
+        $converterclass = $this->get_converter_class_name($question, $quiz, 'missing');
+
+        $qbehaviourupdater = new $converterclass($quiz, $attempt, $question,
+                null, null, $this->logger, $this);
+        $qa = $qbehaviourupdater->supply_missing_qa();
+        $qbehaviourupdater->discard();
+        return $qa;
+    }
+
+    public function convert_question_attempt($quiz, $attempt, $question, $qsession, $qstates) {
+
+        if ($question->qtype == 'random') {
+            list($question, $qstates) = $this->decode_random_attempt($qstates, $question->maxmark);
+            $qsession->questionid = $question->id;
+        }
+
+        $converterclass = $this->get_converter_class_name($question, $quiz, $qsession->id);
+
+        $qbehaviourupdater = new $converterclass($quiz, $attempt, $question, $qsession,
+                $qstates, $this->logger, $this);
+        $qa = $qbehaviourupdater->get_converted_qa();
+        $qbehaviourupdater->discard();
+        return $qa;
+    }
+
+    protected function decode_random_attempt($qstates, $maxmark) {
+        $realquestionid = null;
+        foreach ($qstates as $i => $state) {
+            if (strpos($state->answer, '-') < 6) {
+                // Broken state, skip it.
+                $this->logger->log_assumption("Had to skip brokes state {$state->id}
+                        for question {$state->question}.");
+                unset($qstates[$i]);
+                continue;
+            }
+            list($randombit, $realanswer) = explode('-', $state->answer, 2);
+            $newquestionid = substr($randombit, 6);
+            if ($realquestionid && $realquestionid != $newquestionid) {
+                throw new coding_exception("Question session {$this->qsession->id}
+                        for random question points to two different real questions
+                        {$realquestionid} and {$newquestionid}.");
+            }
+            $qstates[$i]->answer = $realanswer;
+        }
+
+        if (empty($newquestionid)) {
+            // This attempt only had broken states. Set a fake $newquestionid to
+            // prevent a null DB error later.
+            $newquestionid = 0;
+        }
+
+        $newquestion = $this->load_question($newquestionid);
+        $newquestion->maxmark = $maxmark;
+        return array($newquestion, $qstates);
+    }
+
+    public function prepare_to_restore() {
+        $this->logger = new dummy_question_engine_assumption_logger();
+        $this->questionloader = new question_engine_upgrade_question_loader($this->logger);
+    }
+}
+
+
+/**
+ * This class deals with loading (and caching) question definitions during the
+ * question engine upgrade.
+ *
+ * @copyright  2010 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class question_engine_upgrade_question_loader {
+    protected $cache = array();
+    protected $datasetcache = array();
+
+    /** @var base_logger */
+    protected $logger;
+
+    public function __construct($logger) {
+        $this->logger = $logger;
+    }
+
+    protected function load_question($questionid, $quizid) {
+        global $DB;
+
+        if ($quizid) {
+            $question = $DB->get_record_sql("
+                SELECT q.*, slot.maxmark
+                FROM {question} q
+                JOIN {quiz_slots} slot ON slot.questionid = q.id
+                WHERE q.id = ? AND slot.quizid = ?", array($questionid, $quizid));
+        } else {
+            $question = $DB->get_record('question', array('id' => $questionid));
+        }
+
+        if (!$question) {
+            return null;
+        }
+
+        if (empty($question->defaultmark)) {
+            if (!empty($question->defaultgrade)) {
+                $question->defaultmark = $question->defaultgrade;
+            } else {
+                $question->defaultmark = 0;
+            }
+            unset($question->defaultgrade);
+        }
+
+        $qtype = question_bank::get_qtype($question->qtype, false);
+        if ($qtype->name() === 'missingtype') {
+            $this->logger->log_assumption("Dealing with question id {$question->id}
+                    that is of an unknown type {$question->qtype}.");
+            $question->questiontext = '<p>' . get_string('warningmissingtype', 'quiz') .
+                    '</p>' . $question->questiontext;
+        }
+
+        $qtype->get_question_options($question);
+
+        return $question;
+    }
+
+    public function get_question($questionid, $quizid) {
+        if (isset($this->cache[$questionid])) {
+            return $this->cache[$questionid];
+        }
+
+        $question = $this->load_question($questionid, $quizid);
+
+        if (!$question) {
+            $this->logger->log_assumption("Dealing with question id {$questionid}
+                    that was missing from the database.");
+            $question = new stdClass();
+            $question->id = $questionid;
+            $question->qtype = 'deleted';
+            $question->maxmark = 1; // Guess, but that is all we can do.
+            $question->questiontext = get_string('deletedquestiontext', 'qtype_missingtype');
+        }
+
+        $this->cache[$questionid] = $question;
+        return $this->cache[$questionid];
+    }
+
+    public function load_dataset($questionid, $selecteditem) {
+        global $DB;
+
+        if (isset($this->datasetcache[$questionid][$selecteditem])) {
+            return $this->datasetcache[$questionid][$selecteditem];
+        }
+
+        $this->datasetcache[$questionid][$selecteditem] = $DB->get_records_sql_menu('
+                SELECT qdd.name, qdi.value
+                  FROM {question_dataset_items} qdi
+                  JOIN {question_dataset_definitions} qdd ON qdd.id = qdi.definition
+                  JOIN {question_datasets} qd ON qdd.id = qd.datasetdefinition
+                 WHERE qd.question = ?
+                   AND qdi.itemnumber = ?
+                ', array($questionid, $selecteditem));
+        return $this->datasetcache[$questionid][$selecteditem];
+    }
+}
+
+
+/**
+ * Base class for the classes that convert the question-type specific bits of
+ * the attempt data.
+ *
+ * @copyright  2010 The Open University
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+abstract class question_qtype_attempt_updater {
+    /** @var object the question definition data. */
+    protected $question;
+    /** @var question_behaviour_attempt_updater */
+    protected $updater;
+    /** @var question_engine_assumption_logger */
+    protected $logger;
+    /** @var question_engine_attempt_upgrader */
+    protected $qeupdater;
+
+    public function __construct($updater, $question, $logger, $qeupdater) {
+        $this->updater = $updater;
+        $this->question = $question;
+        $this->logger = $logger;
+        $this->qeupdater = $qeupdater;
+    }
+
+    public function discard() {
+        // Help the garbage collector, which seems to be struggling.
+        $this->updater = null;
+        $this->question = null;
+        $this->logger = null;
+        $this->qeupdater = null;
+    }
+
+    protected function to_text($html) {
+        return $this->updater->to_text($html);
+    }
+
+    public function question_summary() {
+        return $this->to_text($this->question->questiontext);
+    }
+
+    public function compare_answers($answer1, $answer2) {
+        return $answer1 == $answer2;
+    }
+
+    public function is_blank_answer($state) {
+        return $state->answer == '';
+    }
+
+    abstract public function right_answer();
+    abstract public function response_summary($state);
+    abstract public function was_answered($state);
+    abstract public function set_first_step_data_elements($state, &$data);
+    abstract public function set_data_elements_for_step($state, &$data);
+    abstract public function supply_missing_first_step_data(&$data);
+}
+
+
+class question_deleted_question_attempt_updater extends question_qtype_attempt_updater {
+    public function right_answer() {
+        return '';
+    }
+
+    public function response_summary($state) {
+        return $state->answer;
+    }
+
+    public function was_answered($state) {
+        return !empty($state->answer);
+    }
+
+    public function set_first_step_data_elements($state, &$data) {
+        $data['upgradedfromdeletedquestion'] = $state->answer;
+    }
+
+    public function supply_missing_first_step_data(&$data) {
+    }
+
+    public function set_data_elements_for_step($state, &$data) {
+        $data['upgradedfromdeletedquestion'] = $state->answer;
+    }
+}
+
+/**
+ * This check verifies that all quiz attempts were upgraded since following
+ * the question engine upgrade in Moodle 2.1.
+ *
+ * Note: This custom check (and its environment.xml declaration) will be safely
+ *       removed once we raise min required Moodle version to be 2.7 or newer.
+ *
+ * @param environment_results object to update, if relevant.
+ * @return environment_results updated results object, or null if this test is not relevant.
+ */
+function quiz_attempts_upgraded(environment_results $result) {
+    global $DB;
+
+    $dbman = $DB->get_manager();
+    $table = new xmldb_table('quiz_attempts');
+    $field = new xmldb_field('needsupgradetonewqe');
+
+    if (!$dbman->table_exists($table) || !$dbman->field_exists($table, $field)) {
+        // DB already upgraded. This test is no longer relevant.
+        return null;
+    }
+
+    if (!$DB->record_exists('quiz_attempts', array('needsupgradetonewqe' => 1))) {
+        // No 1s present in that column means there are no problems.
+        return null;
+    }
+
+    // Only display anything if the admins need to be aware of the problem.
+    $result->setStatus(false);
+    return $result;
+}