first commit

search/engine/solr/classes/document.php

@@ -0,0 +1,208 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Document representation.
+ *
+ * @package    search_solr
+ * @copyright  2015 David Monllao {@link http://www.davidmonllao.com}
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace search_solr;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Respresents a document to index.
+ *
+ * @copyright  2015 David Monllao {@link http://www.davidmonllao.com}
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class document extends \core_search\document {
+    /**
+     * Indicates the file contents were not indexed due to an error.
+     */
+    const INDEXED_FILE_ERROR = -1;
+
+    /**
+     * Indicates the file contents were not indexed due filtering/settings.
+     */
+    const INDEXED_FILE_FALSE = 0;
+
+    /**
+     * Indicates the file contents are indexed with the record.
+     */
+    const INDEXED_FILE_TRUE = 1;
+
+    /**
+     * Any fields that are engine specifc. These are fields that are solely used by a seach engine plugin
+     * for internal purposes.
+     *
+     * @var array
+     */
+    protected static $enginefields = array(
+        'solr_filegroupingid' => array(
+            'type' => 'string',
+            'stored' => true,
+            'indexed' => true
+        ),
+        'solr_fileid' => array(
+            'type' => 'string',
+            'stored' => true,
+            'indexed' => true
+        ),
+        'solr_filecontenthash' => array(
+            'type' => 'string',
+            'stored' => true,
+            'indexed' => true
+        ),
+        // Stores the status of file indexing.
+        'solr_fileindexstatus' => array(
+            'type' => 'int',
+            'stored' => true,
+            'indexed' => true
+        ),
+        // Field to index, but not store, file contents.
+        'solr_filecontent' => array(
+            'type' => 'text',
+            'stored' => false,
+            'indexed' => true,
+            'mainquery' => true
+        )
+    );
+
+    /**
+     * Formats the timestamp according to the search engine needs.
+     *
+     * @param int $timestamp
+     * @return string
+     */
+    public static function format_time_for_engine($timestamp) {
+        return gmdate(\search_solr\engine::DATE_FORMAT, $timestamp);
+    }
+
+    /**
+     * Formats the timestamp according to the search engine needs.
+     *
+     * @param int $timestamp
+     * @return string
+     */
+    public static function format_string_for_engine($string) {
+        // 2^15 default. We could convert this to a setting as is possible to
+        // change the max in solr.
+        return \core_text::str_max_bytes($string, 32766);
+    }
+
+    /**
+     * Returns a timestamp from the value stored in the search engine.
+     *
+     * @param string $time
+     * @return int
+     */
+    public static function import_time_from_engine($time) {
+        return strtotime($time);
+    }
+
+    /**
+     * Overwritten to use HTML (highlighting).
+     *
+     * @return int
+     */
+    protected function get_text_format() {
+        return FORMAT_HTML;
+    }
+
+    /**
+     * Formats a text string coming from the search engine.
+     *
+     * Even if this is called through an external function it is fine to return HTML as
+     * HTML is considered solr's search engine text format. An external function can ask
+     * for raw text, but this just means that it will not pass through format_text, no that
+     * we can not add HTML.
+     *
+     * @param  string $text Text to format
+     * @return string HTML text to be renderer
+     */
+    protected function format_text($text) {
+        // Since we allow output for highlighting, we need to encode html entities.
+        // This ensures plaintext html chars don't become valid html.
+        $out = s($text);
+
+        $startcount = 0;
+        $endcount = 0;
+
+        // Remove end/start pairs that span a few common seperation characters. Allows us to highlight phrases instead of words.
+        $regex = '|'.engine::HIGHLIGHT_END.'([ .,-]{0,3})'.engine::HIGHLIGHT_START.'|';
+        $out = preg_replace($regex, '$1', $out);
+
+        // Now replace our start and end highlight markers.
+        $out = str_replace(engine::HIGHLIGHT_START, '<span class="highlight">', $out, $startcount);
+        $out = str_replace(engine::HIGHLIGHT_END, '</span>', $out, $endcount);
+
+        // This makes sure any highlight tags are balanced, incase truncation or the highlight text contained our markers.
+        while ($startcount > $endcount) {
+            $out .= '</span>';
+            $endcount++;
+        }
+        while ($startcount < $endcount) {
+            $out = '<span class="highlight">' . $out;
+            $endcount++;
+        }
+
+        return parent::format_text($out);
+    }
+
+    /**
+     * Apply any defaults to unset fields before export. Called after document building, but before export.
+     *
+     * Sub-classes of this should make sure to call parent::apply_defaults().
+     */
+    protected function apply_defaults() {
+        parent::apply_defaults();
+
+        // We want to set the solr_filegroupingid to id if it isn't set.
+        if (!isset($this->data['solr_filegroupingid'])) {
+            $this->data['solr_filegroupingid'] = $this->data['id'];
+        }
+    }
+
+    /**
+     * Export the data for the given file in relation to this document.
+     *
+     * @param \stored_file $file The stored file we are talking about.
+     * @return array
+     */
+    public function export_file_for_engine($file) {
+        $data = $this->export_for_engine();
+
+        // Content is index in the main document.
+        unset($data['content']);
+        unset($data['description1']);
+        unset($data['description2']);
+
+        // Going to append the fileid to give it a unique id.
+        $data['id'] = $data['id'].'-solrfile'.$file->get_id();
+        $data['type'] = \core_search\manager::TYPE_FILE;
+        $data['solr_fileid'] = $file->get_id();
+        $data['solr_filecontenthash'] = $file->get_contenthash();
+        $data['solr_fileindexstatus'] = self::INDEXED_FILE_TRUE;
+        $data['title'] = $file->get_filename();
+        $data['modified'] = self::format_time_for_engine($file->get_timemodified());
+
+        return $data;
+    }
+}

search/engine/solr/classes/privacy/provider.php

@@ -0,0 +1,108 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Privacy class for requesting user data.
+ *
+ * @package    search_solr
+ * @copyright  2018 David Monllao
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+namespace search_solr\privacy;
+
+defined('MOODLE_INTERNAL') || die();
+
+use core_privacy\local\metadata\collection;
+use core_privacy\local\request\contextlist;
+use core_privacy\local\request\approved_contextlist;
+use core_privacy\local\request\approved_userlist;
+use core_privacy\local\request\userlist;
+
+/**
+ * Provider for the search_solr plugin.
+ *
+ * @copyright  2018 David Monllao
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class provider implements
+        // This search engine plugin does not store any data itself.
+        // It has no database tables, and it purely acts as a conduit, sending data externally.
+        // This plugin is capable of determining which users have data within it.
+        \core_privacy\local\metadata\provider,
+        \core_privacy\local\request\core_userlist_provider,
+        \core_privacy\local\request\plugin\provider {
+
+    /**
+     * Returns meta data about this system.
+     *
+     * @param   collection $collection The initialised collection to add items to.
+     * @return  collection     A listing of user data stored through this system.
+     */
+    public static function get_metadata(collection $collection): collection {
+        return $collection->add_external_location_link('solr', ['data' => 'privacy:metadata:data'],
+                                                       'privacy:metadata');
+    }
+
+    /**
+     * Get the list of contexts that contain user information for the specified user.
+     *
+     * @param   int $userid The user to search.
+     * @return  contextlist   $contextlist  The contextlist containing the list of contexts used in this plugin.
+     */
+    public static function get_contexts_for_userid(int $userid): contextlist {
+        return new contextlist();
+    }
+
+    /**
+     * Get the list of users who have data within a context.
+     *
+     * @param   userlist    $userlist   The userlist containing the list of users who have data in this context/plugin combination.
+     */
+    public static function get_users_in_context(userlist $userlist) {
+    }
+
+    /**
+     * Export all user data for the specified user, in the specified contexts.
+     *
+     * @param   approved_contextlist $contextlist The approved contexts to export information for.
+     */
+    public static function export_user_data(approved_contextlist $contextlist) {
+    }
+
+    /**
+     * Delete all data for all users in the specified context.
+     *
+     * @param   context $context The specific context to delete data for.
+     */
+    public static function delete_data_for_all_users_in_context(\context $context) {
+    }
+
+    /**
+     * Delete all user data for the specified user, in the specified contexts.
+     *
+     * @param   approved_contextlist $contextlist The approved contexts and user information to delete information for.
+     */
+    public static function delete_data_for_user(approved_contextlist $contextlist) {
+    }
+
+    /**
+     * Delete multiple users within a single context.
+     *
+     * @param   approved_userlist       $userlist The approved context and user information to delete information for.
+     */
+    public static function delete_data_for_users(approved_userlist $userlist) {
+    }
+}

search/engine/solr/classes/schema.php

@@ -0,0 +1,349 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Solr schema manipulation manager.
+ *
+ * @package   search_solr
+ * @copyright 2015 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace search_solr;
+
+defined('MOODLE_INTERNAL') || die();
+
+require_once($CFG->dirroot . '/lib/filelib.php');
+
+/**
+ * Schema class to interact with Solr schema.
+ *
+ * At the moment it only implements create which should be enough for a basic
+ * moodle configuration in Solr.
+ *
+ * @package   search_solr
+ * @copyright 2015 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class schema {
+
+    /**
+     * @var stdClass
+     */
+    protected $config = null;
+
+    /**
+     * cUrl instance.
+     * @var \curl
+     */
+    protected $curl = null;
+
+    /**
+     * An engine instance.
+     * @var engine
+     */
+    protected $engine = null;
+
+    /**
+     * Constructor.
+     *
+     * @param engine $engine Optional engine parameter, if not specified then one will be created
+     * @throws \moodle_exception
+     * @return void
+     */
+    public function __construct(engine $engine = null) {
+        if (!$this->config = get_config('search_solr')) {
+            throw new \moodle_exception('missingconfig', 'search_solr');
+        }
+
+        if (empty($this->config->server_hostname) || empty($this->config->indexname)) {
+            throw new \moodle_exception('missingconfig', 'search_solr');
+        }
+
+        $this->engine = $engine ?? new engine();
+        $this->curl = $this->engine->get_curl_object();
+
+        // HTTP headers.
+        $this->curl->setHeader('Content-type: application/json');
+    }
+
+    /**
+     * Can setup be executed against the configured server.
+     *
+     * @return true|string True or error message.
+     */
+    public function can_setup_server() {
+
+        $status = $this->engine->is_server_configured();
+        if ($status !== true) {
+            return $status;
+        }
+
+        // At this stage we know that the server is properly configured with a valid host:port and indexname.
+        // We're not too concerned about repeating the SolrClient::system() call (already called in
+        // is_server_configured) because this is just a setup script.
+        if ($this->engine->get_solr_major_version() < 5) {
+            // Schema setup script only available for 5.0 onwards.
+            return get_string('schemasetupfromsolr5', 'search_solr');
+        }
+
+        return true;
+    }
+
+    /**
+     * Setup solr stuff required by moodle.
+     *
+     * @param  bool $checkexisting Whether to check if the fields already exist or not
+     * @return bool
+     */
+    public function setup($checkexisting = true) {
+        $fields = \search_solr\document::get_default_fields_definition();
+
+        // Field id is already there.
+        unset($fields['id']);
+
+        $this->check_index();
+
+        $return = $this->add_fields($fields, $checkexisting);
+
+        // Tell the engine we are now using the latest schema version.
+        $this->engine->record_applied_schema_version(document::SCHEMA_VERSION);
+
+        return $return;
+    }
+
+    /**
+     * Checks the schema is properly set up.
+     *
+     * @throws \moodle_exception
+     * @return void
+     */
+    public function validate_setup() {
+        $fields = \search_solr\document::get_default_fields_definition();
+
+        // Field id is already there.
+        unset($fields['id']);
+
+        $this->check_index();
+        $this->validate_fields($fields, true);
+    }
+
+    /**
+     * Checks if the index is ready, triggers an exception otherwise.
+     *
+     * @throws \moodle_exception
+     * @return void
+     */
+    protected function check_index() {
+
+        // Check that the server is available and the index exists.
+        $url = $this->engine->get_connection_url('/select?wt=json');
+        $result = $this->curl->get($url);
+        if ($this->curl->error) {
+            throw new \moodle_exception('connectionerror', 'search_solr');
+        }
+        if ($this->curl->info['http_code'] === 404) {
+            throw new \moodle_exception('connectionerror', 'search_solr');
+        }
+    }
+
+    /**
+     * Adds the provided fields to Solr schema.
+     *
+     * Intentionally separated from create(), it can be called to add extra fields.
+     * fields separately.
+     *
+     * @throws \coding_exception
+     * @throws \moodle_exception
+     * @param  array $fields \core_search\document::$requiredfields format
+     * @param  bool $checkexisting Whether to check if the fields already exist or not
+     * @return bool
+     */
+    protected function add_fields($fields, $checkexisting = true) {
+
+        if ($checkexisting) {
+            // Check that non of them exists.
+            $this->validate_fields($fields, false);
+        }
+
+        $url = $this->engine->get_connection_url('/schema');
+
+        // Add all fields.
+        foreach ($fields as $fieldname => $data) {
+
+            if (!isset($data['type']) || !isset($data['stored']) || !isset($data['indexed'])) {
+                throw new \coding_exception($fieldname . ' does not define all required field params: type, stored and indexed.');
+            }
+            $type = $this->doc_field_to_solr_field($data['type']);
+
+            // Changing default multiValued value to false as we want to match values easily.
+            $params = array(
+                'add-field' => array(
+                    'name' => $fieldname,
+                    'type' => $type,
+                    'stored' => $data['stored'],
+                    'multiValued' => false,
+                    'indexed' => $data['indexed']
+                )
+            );
+            $results = $this->curl->post($url, json_encode($params));
+
+            // We only validate if we are interested on it.
+            if ($checkexisting) {
+                if ($this->curl->error) {
+                    throw new \moodle_exception('errorcreatingschema', 'search_solr', '', $this->curl->error);
+                }
+                $this->validate_add_field_result($results);
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * Checks if the schema existing fields are properly set, triggers an exception otherwise.
+     *
+     * @throws \moodle_exception
+     * @param array $fields
+     * @param bool $requireexisting Require the fields to exist, otherwise exception.
+     * @return void
+     */
+    protected function validate_fields(&$fields, $requireexisting = false) {
+        global $CFG;
+
+        foreach ($fields as $fieldname => $data) {
+            $url = $this->engine->get_connection_url('/schema/fields/' . $fieldname);
+            $results = $this->curl->get($url);
+
+            if ($this->curl->error) {
+                throw new \moodle_exception('errorcreatingschema', 'search_solr', '', $this->curl->error);
+            }
+
+            if (!$results) {
+                throw new \moodle_exception('errorcreatingschema', 'search_solr', '', get_string('nodatafromserver', 'search_solr'));
+            }
+            $results = json_decode($results);
+
+            if ($requireexisting && !empty($results->error) && $results->error->code === 404) {
+                $a = new \stdClass();
+                $a->fieldname = $fieldname;
+                $a->setupurl = $CFG->wwwroot . '/search/engine/solr/setup_schema.php';
+                throw new \moodle_exception('errorvalidatingschema', 'search_solr', '', $a);
+            }
+
+            // The field should not exist so we only accept 404 errors.
+            if (empty($results->error) || (!empty($results->error) && $results->error->code !== 404)) {
+                if (!empty($results->error)) {
+                    throw new \moodle_exception('errorcreatingschema', 'search_solr', '', $results->error->msg);
+                } else {
+                    // All these field attributes are set when fields are added through this script and should
+                    // be returned and match the defined field's values.
+
+                    $expectedsolrfield = $this->doc_field_to_solr_field($data['type']);
+                    if (empty($results->field) || !isset($results->field->type) ||
+                            !isset($results->field->multiValued) || !isset($results->field->indexed) ||
+                            !isset($results->field->stored)) {
+
+                        throw new \moodle_exception('errorcreatingschema', 'search_solr', '',
+                            get_string('schemafieldautocreated', 'search_solr', $fieldname));
+
+                    } else if ($results->field->type !== $expectedsolrfield ||
+                            $results->field->multiValued !== false ||
+                            $results->field->indexed !== $data['indexed'] ||
+                            $results->field->stored !== $data['stored']) {
+
+                        throw new \moodle_exception('errorcreatingschema', 'search_solr', '',
+                            get_string('schemafieldautocreated', 'search_solr', $fieldname));
+                    } else {
+                        // The field already exists and it is properly defined, no need to create it.
+                        unset($fields[$fieldname]);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Checks that the field results do not contain errors.
+     *
+     * @throws \moodle_exception
+     * @param string $results curl response body
+     * @return void
+     */
+    protected function validate_add_field_result($result) {
+
+        if (!$result) {
+            throw new \moodle_exception('errorcreatingschema', 'search_solr', '', get_string('nodatafromserver', 'search_solr'));
+        }
+
+        $results = json_decode($result);
+        if (!$results) {
+            if (is_scalar($result)) {
+                $errormsg = $result;
+            } else {
+                $errormsg = json_encode($result);
+            }
+            throw new \moodle_exception('errorcreatingschema', 'search_solr', '', $errormsg);
+        }
+
+        // It comes as error when fetching fields data.
+        if (!empty($results->error)) {
+            throw new \moodle_exception('errorcreatingschema', 'search_solr', '', $results->error);
+        }
+
+        // It comes as errors when adding fields.
+        if (!empty($results->errors)) {
+
+            // We treat this error separately.
+            $errorstr = '';
+            foreach ($results->errors as $error) {
+                $errorstr .= implode(', ', $error->errorMessages);
+            }
+            throw new \moodle_exception('errorcreatingschema', 'search_solr', '', $errorstr);
+        }
+
+    }
+
+    /**
+     * Returns the solr field type from the document field type string.
+     *
+     * @param string $datatype
+     * @return string
+     */
+    private function doc_field_to_solr_field($datatype) {
+        $type = $datatype;
+
+        $solrversion = $this->engine->get_solr_major_version();
+
+        switch($datatype) {
+            case 'text':
+                $type = 'text_general';
+                break;
+            case 'int':
+                if ($solrversion >= 7) {
+                    $type = 'pint';
+                }
+                break;
+            case 'tdate':
+                if ($solrversion >= 7) {
+                    $type = 'pdate';
+                }
+                break;
+        }
+
+        return $type;
+    }
+}