first commit

lib/filestorage/file_archive.php

@@ -0,0 +1,215 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+
+/**
+ * Abstraction of general file archives.
+ *
+ * @package   core_files
+ * @copyright 2008 Petr Skoda (http://skodak.org)
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Each file archive type must extend this class.
+ *
+ * @package   core_files
+ * @copyright 2008 Petr Skoda (http://skodak.org)
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+abstract class file_archive implements Iterator {
+
+    /** Open archive if exists, fail if does not exist. */
+    const OPEN      = 1;
+
+    /** Open archive if exists, create if does not. */
+    const CREATE    = 2;
+
+    /** Always create new archive */
+    const OVERWRITE = 4;
+
+    /** @var string Encoding of file names - windows usually expects DOS single-byte charset*/
+    protected $encoding = 'utf-8';
+
+    /**
+     * Open or create archive (depending on $mode).
+     *
+     * @param string $archivepathname archive path name
+     * @param int $mode OPEN, CREATE or OVERWRITE constant
+     * @param string $encoding archive local paths encoding
+     * @return bool success
+     */
+    abstract public function open($archivepathname, $mode=file_archive::CREATE, $encoding='utf-8');
+
+    /**
+     * Close archive.
+     *
+     * @return bool success
+     */
+    abstract public function close();
+
+    /**
+     * Returns file stream for reading of content.
+     *
+     * @param int $index index of file
+     * @return stream|bool stream or false if error
+     */
+    abstract public function get_stream($index);
+
+    /**
+     * Returns file information.
+     *
+     * @param int $index index of file
+     * @return stdClass|bool object or false if error
+     */
+    abstract public function get_info($index);
+
+    /**
+     * Returns array of info about all files in archive.
+     *
+     * @return array of file infos
+     */
+    abstract public function list_files();
+
+    /**
+     * Returns number of files in archive.
+     *
+     * @return int number of files
+     */
+    abstract public function count();
+
+    /**
+     * Add file into archive.
+     *
+     * @param string $localname name of file in archive
+     * @param string $pathname location of file
+     * @return bool success
+     */
+    abstract public function add_file_from_pathname($localname, $pathname);
+
+    /**
+     * Add content of string into archive.
+     *
+     * @param string $localname name of file in archive
+     * @param string $contents contents
+     * @return bool success
+     */
+    abstract public function add_file_from_string($localname, $contents);
+
+    /**
+     * Add empty directory into archive.
+     *
+     * @param string $localname name of file in archive
+     * @return bool success
+     */
+    abstract public function add_directory($localname);
+
+    /**
+     * Tries to convert $localname into another encoding,
+     * please note that it may fail really badly.
+     *
+     * @param string $localname name of file in utf-8 encoding
+     * @return string
+     */
+    protected function mangle_pathname($localname) {
+        if ($this->encoding === 'utf-8') {
+            return $localname;
+        }
+
+        $converted = core_text::convert($localname, 'utf-8', $this->encoding);
+        $original  = core_text::convert($converted, $this->encoding, 'utf-8');
+
+        if ($original === $localname) {
+            $result = $converted;
+
+        } else {
+            // try ascii conversion
+            $converted2 = core_text::specialtoascii($localname);
+            $converted2 = core_text::convert($converted2, 'utf-8', $this->encoding);
+            $original2  = core_text::convert($converted, $this->encoding, 'utf-8');
+
+            if ($original2 === $localname) {
+                //this looks much better
+                $result = $converted2;
+            } else {
+                //bad luck - the file name may not be usable at all
+                $result = $converted;
+            }
+        }
+
+        $result = preg_replace('/\.\.+\//', '', $result); // Cleanup any potential ../ transversal (any number of dots).
+        $result = preg_replace('/\.\.+/', '.', $result); // Join together any number of consecutive dots.
+        $result = ltrim($result); // no leading /
+
+        if ($result === '.') {
+            $result = '';
+        }
+
+        return $result;
+    }
+
+    /**
+     * Tries to convert $localname into utf-8
+     * please note that it may fail really badly.
+     * The resulting file name is cleaned.
+     *
+     * @param string $localname name of file in $this->encoding
+     * @return string in utf-8
+     */
+    protected function unmangle_pathname($localname) {
+        $result = str_replace('\\', '/', $localname); // no MS \ separators
+        $result = ltrim($result, '/');                // no leading /
+
+        if ($this->encoding !== 'utf-8') {
+            $result = core_text::convert($result, $this->encoding, 'utf-8');
+        }
+
+        return clean_param($result, PARAM_PATH);
+    }
+
+    /**
+     * Returns current file info.
+     * @return object
+     */
+    //public abstract function current();
+
+    /**
+     * Returns the index of current file.
+     * @return int current file index
+     */
+    //public abstract function key();
+
+    /**
+     * Moves forward to next file.
+     * @return void
+     */
+    //public abstract function next();
+
+    /**
+     * Rewinds back to the first file.
+     * @return void
+     */
+    //public abstract function rewind();
+
+    /**
+     * Did we reach the end?
+     * @return boolean
+     */
+    //public abstract function valid();
+
+}

lib/filestorage/file_exceptions.php

@@ -0,0 +1,146 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * File handling related exceptions.
+ *
+ * @package   core_files
+ * @copyright 2008 Petr Skoda (http://skodak.org)
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Basic file related exception class.
+ *
+ * @package   core_files
+ * @category  files
+ * @copyright 2008 Petr Skoda (http://skodak.org)
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class file_exception extends moodle_exception {
+    /**
+     * Constructor
+     *
+     * @param string $errorcode error code
+     * @param mixed $a Extra words and phrases that might be required in the error string
+     * @param string $debuginfo optional debugging information
+     */
+    function __construct($errorcode, $a=NULL, $debuginfo = NULL) {
+        parent::__construct($errorcode, '', '', $a, $debuginfo);
+    }
+}
+
+/**
+ * Cannot create file exception.
+ *
+ * @package   core_files
+ * @category  files
+ * @copyright 2008 Petr Skoda (http://skodak.org)
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class stored_file_creation_exception extends file_exception {
+    /**
+     * Constructor.
+     *
+     * @param int $contextid context ID
+     * @param string $component component
+     * @param string $filearea file area
+     * @param int $itemid item ID
+     * @param string $filepath file path
+     * @param string $filename file name
+     * @param string $debuginfo extra debug info
+     */
+    function __construct($contextid, $component, $filearea, $itemid, $filepath, $filename, $debuginfo = null) {
+        $a = new stdClass();
+        $a->contextid = $contextid;
+        $a->component = $component;
+        $a->filearea  = $filearea;
+        $a->itemid    = $itemid;
+        $a->filepath  = $filepath;
+        $a->filename  = $filename;
+        parent::__construct('storedfilenotcreated', $a, $debuginfo);
+    }
+}
+
+/**
+ * No file access exception.
+ *
+ * @package   core_files
+ * @category  files
+ * @copyright 2008 Petr Skoda (http://skodak.org)
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class file_access_exception extends file_exception {
+    /**
+     * Constructor.
+     *
+     * @param string $debuginfo extra debug info
+     */
+    public function __construct($debuginfo = null) {
+        parent::__construct('nopermissions', null, $debuginfo);
+    }
+}
+
+/**
+ * Hash file content problem exception.
+ *
+ * @package   core_files
+ * @category  files
+ * @copyright 2008 Petr Skoda (http://skodak.org)
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class file_pool_content_exception extends file_exception {
+    /**
+     * Constructor.
+     *
+     * @param string $contenthash content hash
+     * @param string $debuginfo extra debug info
+     */
+    public function __construct($contenthash, $debuginfo = null) {
+        parent::__construct('hashpoolproblem', $contenthash, $debuginfo);
+    }
+}
+
+
+/**
+ * Problem with records in the {files_reference} table.
+ *
+ * @package   core_files
+ * @category  files
+ * @copyright 2012 David Mudrak <david@moodle.com>
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class file_reference_exception extends file_exception {
+    /**
+     * Constructor.
+     *
+     * @param ?int $repositoryid the id of the repository that provides the referenced file
+     * @param string $reference the information for the repository to locate the file
+     * @param int|null $referencefileid the id of the record in {files_reference} if known
+     * @param int|null $fileid the id of the referrer's record in {files} if known
+     * @param string|null $debuginfo extra debug info
+     */
+    function __construct($repositoryid, $reference, $referencefileid=null, $fileid=null, $debuginfo=null) {
+        $a = new stdClass();
+        $a->repositoryid = $repositoryid;
+        $a->reference = $reference;
+        $a->referencefileid = $referencefileid;
+        $a->fileid = $fileid;
+        parent::__construct('filereferenceproblem', $a, $debuginfo);
+    }
+}

lib/filestorage/file_packer.php

@@ -0,0 +1,128 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Abstraction of general file packer.
+ *
+ * @package   core_files
+ * @copyright 2008 Petr Skoda (http://skodak.org)
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Abstract class for archiving of files.
+ *
+ * @package   core_files
+ * @copyright 2008 Petr Skoda (http://skodak.org)
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+abstract class file_packer {
+    /**
+     * Archive files and store the result in file storage.
+     *
+     * The key of the $files array is always the path within the archive, e.g.
+     * 'folder/subfolder/file.txt'. There are several options for the values of
+     * the array:
+     * - null = this entry represents a directory, so no file
+     * - string = full path to file within operating system filesystem
+     * - stored_file = file within Moodle filesystem
+     * - array with one string element = use in-memory string for file content
+     *
+     * For the string (OS path) and stored_file (Moodle filesystem) cases, you
+     * can specify a directory instead of a file to recursively include all files
+     * within this directory.
+     *
+     * @param array $files Array of files to archive
+     * @param int $contextid context ID
+     * @param string $component component
+     * @param string $filearea file area
+     * @param int $itemid item ID
+     * @param string $filepath file path
+     * @param string $filename file name
+     * @param int $userid user ID
+     * @param bool $ignoreinvalidfiles true means ignore missing or invalid files, false means abort on any error
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return stored_file|bool false if error stored_file instance if ok
+     */
+    abstract public function archive_to_storage(array $files, $contextid,
+            $component, $filearea, $itemid, $filepath, $filename,
+            $userid = NULL, $ignoreinvalidfiles=true, file_progress $progress = null);
+
+    /**
+     * Archive files and store the result in os file.
+     *
+     * The key of the $files array is always the path within the archive, e.g.
+     * 'folder/subfolder/file.txt'. There are several options for the values of
+     * the array:
+     * - null = this entry represents a directory, so no file
+     * - string = full path to file within operating system filesystem
+     * - stored_file = file within Moodle filesystem
+     * - array with one string element = use in-memory string for file content
+     *
+     * For the string (OS path) and stored_file (Moodle filesystem) cases, you
+     * can specify a directory instead of a file to recursively include all files
+     * within this directory.
+     *
+     * @param array $files array with zip paths as keys (archivepath=>ospathname or archivepath=>stored_file)
+     * @param string $archivefile path to target zip file
+     * @param bool $ignoreinvalidfiles true means ignore missing or invalid files, false means abort on any error
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return bool true if file created, false if not
+     */
+    abstract public function archive_to_pathname(array $files, $archivefile,
+            $ignoreinvalidfiles=true, file_progress $progress = null);
+
+    /**
+     * Extract file to given file path (real OS filesystem), existing files are overwritten.
+     *
+     * @param stored_file|string $archivefile full pathname of zip file or stored_file instance
+     * @param string $pathname target directory
+     * @param array $onlyfiles only extract files present in the array
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @param bool $returnbool Whether to return a basic true/false indicating error state, or full per-file error
+     * details.
+     * @return array|bool list of processed files; false if error
+     */
+    abstract public function extract_to_pathname($archivefile, $pathname,
+            array $onlyfiles = NULL, file_progress $progress = null, $returnbool = false);
+
+    /**
+     * Extract file to given file path (real OS filesystem), existing files are overwritten.
+     *
+     * @param string|stored_file $archivefile full pathname of zip file or stored_file instance
+     * @param int $contextid context ID
+     * @param string $component component
+     * @param string $filearea file area
+     * @param int $itemid item ID
+     * @param string $pathbase file path
+     * @param int $userid user ID
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return array|bool list of processed files; false if error
+     */
+    abstract public function extract_to_storage($archivefile, $contextid,
+            $component, $filearea, $itemid, $pathbase, $userid = NULL,
+            file_progress $progress = null);
+
+    /**
+     * Returns array of info about all files in archive.
+     *
+     * @param string|file_archive $archivefile
+     * @return array of file infos
+     */
+    abstract public function list_files($archivefile);
+}

lib/filestorage/file_progress.php

@@ -0,0 +1,71 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Simple interface for receiving progress during long-running file
+ * operations.
+ *
+ * In some cases progress can be reported precisely. In other cases,
+ * progress is indeterminate which means that the progress function is called
+ * periodically but without information on completion.
+ *
+ * @package core_files
+ * @copyright 2013 The Open University
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+interface file_progress {
+    /**
+     * @var int Constant used for indeterminate progress.
+     */
+    const INDETERMINATE = -1;
+
+    /**
+     * Called during a file processing operation that reports progress.
+     *
+     * This function will be called periodically during the operation, assuming
+     * it is successful.
+     *
+     * The $max value will be the same for each call to progress() within an
+     * operation.
+     *
+     * If numbers (rather than INDETERMINATE) are provided, then:
+     * - The $progress value will either be the same as last call, or increased
+     *   by some value (not necessarily 1)
+     * - The $progress value will be less than or equal to the $max value.
+     *
+     * There is no guarantee that this function will be called for every value
+     * in the range, or that it will be called with $progress == $max.
+     *
+     * The function may be called very frequently (e.g. after each file) or
+     * quite rarely (e.g. after each large file).
+     *
+     * When creating an implementation of this function, you probably want to
+     * do the following:
+     *
+     * 1. Check the current time and do not do anything if it's less than a
+     *    second since the last time you reported something.
+     * 2. Update the PHP timeout (i.e. set it back to 2 minutes or whatever)
+     *    so that the system will not time out.
+     * 3. If the progress is unchanged since last second, still display some
+     *    output to the user. (Setting PHP timeout is not sufficient; some
+     *    front-end servers require that data is output to the browser every
+     *    minute or so, or they will time out on their own.)
+     *
+     * @param int $progress Current progress, or INDETERMINATE if unknown
+     * @param int $max Max progress, or INDETERMINATE if unknown
+     */
+    public function progress($progress = self::INDETERMINATE, $max = self::INDETERMINATE);
+}

lib/filestorage/file_system.php

@@ -0,0 +1,676 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+use Psr\Http\Message\StreamInterface;
+
+/**
+ * File system class used for low level access to real files in filedir.
+ *
+ * @package   core_files
+ * @category  files
+ * @copyright 2017 Andrew Nicols <andrew@nicols.co.uk>
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+abstract class file_system {
+
+    /**
+     * Output the content of the specified stored file.
+     *
+     * Note, this is different to get_content() as it uses the built-in php
+     * readfile function which is more efficient.
+     *
+     * @param stored_file $file The file to serve.
+     * @return void
+     */
+    public function readfile(stored_file $file) {
+        if ($this->is_file_readable_locally_by_storedfile($file, false)) {
+            $path = $this->get_local_path_from_storedfile($file, false);
+        } else {
+            $path = $this->get_remote_path_from_storedfile($file);
+        }
+        if (readfile_allow_large($path, $file->get_filesize()) === false) {
+            throw new file_exception('storedfilecannotreadfile', $file->get_filename());
+        }
+    }
+
+    /**
+     * Get the full path on disk for the specified stored file.
+     *
+     * Note: This must return a consistent path for the file's contenthash
+     * and the path _will_ be in a standard local format.
+     * Streamable paths will not work.
+     * A local copy of the file _will_ be fetched if $fetchifnotfound is tree.
+     *
+     * The $fetchifnotfound allows you to determine the expected path of the file.
+     *
+     * @param stored_file $file The file to serve.
+     * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
+     * @return string full path to pool file with file content
+     */
+    public function get_local_path_from_storedfile(stored_file $file, $fetchifnotfound = false) {
+        return $this->get_local_path_from_hash($file->get_contenthash(), $fetchifnotfound);
+    }
+
+    /**
+     * Get a remote filepath for the specified stored file.
+     *
+     * This is typically either the same as the local filepath, or it is a streamable resource.
+     *
+     * See https://secure.php.net/manual/en/wrappers.php for further information on valid wrappers.
+     *
+     * @param stored_file $file The file to serve.
+     * @return string full path to pool file with file content
+     */
+    public function get_remote_path_from_storedfile(stored_file $file) {
+        return $this->get_remote_path_from_hash($file->get_contenthash(), false);
+    }
+
+    /**
+     * Get the full path for the specified hash, including the path to the filedir.
+     *
+     * Note: This must return a consistent path for the file's contenthash
+     * and the path _will_ be in a standard local format.
+     * Streamable paths will not work.
+     * A local copy of the file _will_ be fetched if $fetchifnotfound is tree.
+     *
+     * The $fetchifnotfound allows you to determine the expected path of the file.
+     *
+     * @param string $contenthash The content hash
+     * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
+     * @return string The full path to the content file
+     */
+    abstract protected function get_local_path_from_hash($contenthash, $fetchifnotfound = false);
+
+    /**
+     * Get the full path for the specified hash, including the path to the filedir.
+     *
+     * This is typically either the same as the local filepath, or it is a streamable resource.
+     *
+     * See https://secure.php.net/manual/en/wrappers.php for further information on valid wrappers.
+     *
+     * @param string $contenthash The content hash
+     * @return string The full path to the content file
+     */
+    abstract protected function get_remote_path_from_hash($contenthash);
+
+    /**
+     * Determine whether the file is present on the file system somewhere.
+     * A local copy of the file _will_ be fetched if $fetchifnotfound is tree.
+     *
+     * The $fetchifnotfound allows you to determine the expected path of the file.
+     *
+     * @param stored_file $file The file to ensure is available.
+     * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
+     * @return bool
+     */
+    public function is_file_readable_locally_by_storedfile(stored_file $file, $fetchifnotfound = false) {
+        if (!$file->get_filesize()) {
+            // Files with empty size are either directories or empty.
+            // We handle these virtually.
+            return true;
+        }
+
+        // Check to see if the file is currently readable.
+        $path = $this->get_local_path_from_storedfile($file, $fetchifnotfound);
+        if (is_readable($path)) {
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * Determine whether the file is present on the local file system somewhere.
+     *
+     * @param stored_file $file The file to ensure is available.
+     * @return bool
+     */
+    public function is_file_readable_remotely_by_storedfile(stored_file $file) {
+        if (!$file->get_filesize()) {
+            // Files with empty size are either directories or empty.
+            // We handle these virtually.
+            return true;
+        }
+
+        $path = $this->get_remote_path_from_storedfile($file, false);
+        if (is_readable($path)) {
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * Determine whether the file is present on the file system somewhere given
+     * the contenthash.
+     *
+     * @param string $contenthash The contenthash of the file to check.
+     * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
+     * @return bool
+     */
+    public function is_file_readable_locally_by_hash($contenthash, $fetchifnotfound = false) {
+        if ($contenthash === file_storage::hash_from_string('')) {
+            // Files with empty size are either directories or empty.
+            // We handle these virtually.
+            return true;
+        }
+
+        // This is called by file_storage::content_exists(), and in turn by the repository system.
+        $path = $this->get_local_path_from_hash($contenthash, $fetchifnotfound);
+
+        // Note - it is not possible to perform a content recovery safely from a hash alone.
+        return is_readable($path);
+    }
+
+    /**
+     * Determine whether the file is present locally on the file system somewhere given
+     * the contenthash.
+     *
+     * @param string $contenthash The contenthash of the file to check.
+     * @return bool
+     */
+    public function is_file_readable_remotely_by_hash($contenthash) {
+        if ($contenthash === file_storage::hash_from_string('')) {
+            // Files with empty size are either directories or empty.
+            // We handle these virtually.
+            return true;
+        }
+
+        $path = $this->get_remote_path_from_hash($contenthash, false);
+
+        // Note - it is not possible to perform a content recovery safely from a hash alone.
+        return is_readable($path);
+    }
+
+    /**
+     * Copy content of file to given pathname.
+     *
+     * @param stored_file $file The file to be copied
+     * @param string $target real path to the new file
+     * @return bool success
+     */
+    abstract public function copy_content_from_storedfile(stored_file $file, $target);
+
+    /**
+     * Remove the file with the specified contenthash.
+     *
+     * Note, if overriding this function, you _must_ check that the file is
+     * no longer in use - see {check_file_usage}.
+     *
+     * DO NOT call directly - reserved for core!!
+     *
+     * @param string $contenthash
+     */
+    abstract public function remove_file($contenthash);
+
+    /**
+     * Check whether a file is removable.
+     *
+     * This must be called prior to file removal.
+     *
+     * @param string $contenthash
+     * @return bool
+     */
+    protected static function is_file_removable($contenthash) {
+        global $DB;
+
+        if ($contenthash === file_storage::hash_from_string('')) {
+            // No need to delete files without content.
+            return false;
+        }
+
+        // Note: This section is critical - in theory file could be reused at the same time, if this
+        // happens we can still recover the file from trash.
+        // Technically this is the responsibility of the file_storage API, but as this method is public, we go belt-and-braces.
+        if ($DB->record_exists('files', array('contenthash' => $contenthash))) {
+            // File content is still used.
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * Get the content of the specified stored file.
+     *
+     * Generally you will probably want to use readfile() to serve content,
+     * and where possible you should see if you can use
+     * get_content_file_handle and work with the file stream instead.
+     *
+     * @param stored_file $file The file to retrieve
+     * @return string The full file content
+     */
+    public function get_content(stored_file $file) {
+        if (!$file->get_filesize()) {
+            // Directories are empty. Empty files are not worth fetching.
+            return '';
+        }
+
+        $source = $this->get_remote_path_from_storedfile($file);
+        return file_get_contents($source);
+    }
+
+    /**
+     * List contents of archive.
+     *
+     * @param stored_file $file The archive to inspect
+     * @param file_packer $packer file packer instance
+     * @return array of file infos
+     */
+    public function list_files($file, file_packer $packer) {
+        $archivefile = $this->get_local_path_from_storedfile($file, true);
+        return $packer->list_files($archivefile);
+    }
+
+    /**
+     * Extract file to given file path (real OS filesystem), existing files are overwritten.
+     *
+     * @param stored_file $file The archive to inspect
+     * @param file_packer $packer File packer instance
+     * @param string $pathname Target directory
+     * @param file_progress $progress progress indicator callback or null if not required
+     * @return array|bool List of processed files; false if error
+     */
+    public function extract_to_pathname(stored_file $file, file_packer $packer, $pathname, file_progress $progress = null) {
+        $archivefile = $this->get_local_path_from_storedfile($file, true);
+        return $packer->extract_to_pathname($archivefile, $pathname, null, $progress);
+    }
+
+    /**
+     * Extract file to given file path (real OS filesystem), existing files are overwritten.
+     *
+     * @param stored_file $file The archive to inspect
+     * @param file_packer $packer file packer instance
+     * @param int $contextid context ID
+     * @param string $component component
+     * @param string $filearea file area
+     * @param int $itemid item ID
+     * @param string $pathbase path base
+     * @param int $userid user ID
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return array|bool list of processed files; false if error
+     */
+    public function extract_to_storage(stored_file $file, file_packer $packer, $contextid,
+            $component, $filearea, $itemid, $pathbase, $userid = null, file_progress $progress = null) {
+
+        // Since we do not know which extractor we have, and whether it supports remote paths, use a local path here.
+        $archivefile = $this->get_local_path_from_storedfile($file, true);
+        return $packer->extract_to_storage($archivefile, $contextid,
+                $component, $filearea, $itemid, $pathbase, $userid, $progress);
+    }
+
+    /**
+     * Add file/directory into archive.
+     *
+     * @param stored_file $file The file to archive
+     * @param file_archive $filearch file archive instance
+     * @param string $archivepath pathname in archive
+     * @return bool success
+     */
+    public function add_storedfile_to_archive(stored_file $file, file_archive $filearch, $archivepath) {
+        if ($file->is_directory()) {
+            return $filearch->add_directory($archivepath);
+        } else {
+            // Since we do not know which extractor we have, and whether it supports remote paths, use a local path here.
+            return $filearch->add_file_from_pathname($archivepath, $this->get_local_path_from_storedfile($file, true));
+        }
+    }
+
+    /**
+     * Adds this file path to a curl request (POST only).
+     *
+     * @param stored_file $file The file to add to the curl request
+     * @param curl $curlrequest The curl request object
+     * @param string $key What key to use in the POST request
+     * @return void
+     * This needs the fullpath for the storedfile :/
+     * Can this be achieved in some other fashion?
+     */
+    public function add_to_curl_request(stored_file $file, &$curlrequest, $key) {
+        // Note: curl_file_create does not work with remote paths.
+        $path = $this->get_local_path_from_storedfile($file, true);
+        $curlrequest->_tmp_file_post_params[$key] = curl_file_create($path, null, $file->get_filename());
+    }
+
+    /**
+     * Returns information about image.
+     * Information is determined from the file content
+     *
+     * @param stored_file $file The file to inspect
+     * @return mixed array with width, height and mimetype; false if not an image
+     */
+    public function get_imageinfo(stored_file $file) {
+        if (!$this->is_image_from_storedfile($file)) {
+            return false;
+        }
+
+        $hash = $file->get_contenthash();
+        $cache = cache::make('core', 'file_imageinfo');
+        $info = $cache->get($hash);
+        if ($info !== false) {
+            return $info;
+        }
+
+        // Whilst get_imageinfo_from_path can use remote paths, it must download the entire file first.
+        // It is more efficient to use a local file when possible.
+        $info = $this->get_imageinfo_from_path($this->get_local_path_from_storedfile($file, true));
+        $cache->set($hash, $info);
+        return $info;
+    }
+
+    /**
+     * Attempt to determine whether the specified file is likely to be an
+     * image.
+     * Since this relies upon the mimetype stored in the files table, there
+     * may be times when this information is not 100% accurate.
+     *
+     * @param stored_file $file The file to check
+     * @return bool
+     */
+    public function is_image_from_storedfile(stored_file $file) {
+        if (!$file->get_filesize()) {
+            // An empty file cannot be an image.
+            return false;
+        }
+
+        $mimetype = $file->get_mimetype();
+        if (!preg_match('|^image/|', $mimetype)) {
+            // The mimetype does not include image.
+            return false;
+        }
+
+        // If it looks like an image, and it smells like an image, perhaps it's an image!
+        return true;
+    }
+
+    /**
+     * Returns image information relating to the specified path or URL.
+     *
+     * @param string $path The full path of the image file.
+     * @return array|bool array that containing width, height, and mimetype or false if cannot get the image info.
+     */
+    protected function get_imageinfo_from_path($path) {
+        $imagemimetype = file_storage::mimetype_from_file($path);
+        $issvgimage = file_is_svg_image_from_mimetype($imagemimetype);
+
+        if (!$issvgimage) {
+            $imageinfo = getimagesize($path);
+            if (!is_array($imageinfo)) {
+                return false; // Nothing to process, the file was not recognised as image by GD.
+            }
+            $image = [
+                    'width' => $imageinfo[0],
+                    'height' => $imageinfo[1],
+                    'mimetype' => image_type_to_mime_type($imageinfo[2]),
+            ];
+        } else {
+            // Since SVG file is actually an XML file, GD cannot handle.
+            $svgcontent = @simplexml_load_file($path);
+            if (!$svgcontent) {
+                // Cannot parse the file.
+                return false;
+            }
+            $svgattrs = $svgcontent->attributes();
+
+            if (!empty($svgattrs->viewBox)) {
+                // We have viewBox.
+                $viewboxval = explode(' ', $svgattrs->viewBox);
+                $width = intval($viewboxval[2]);
+                $height = intval($viewboxval[3]);
+            } else {
+                // Get the width.
+                if (!empty($svgattrs->width) && intval($svgattrs->width) > 0) {
+                    $width = intval($svgattrs->width);
+                } else {
+                    // Default width.
+                    $width = 800;
+                }
+                // Get the height.
+                if (!empty($svgattrs->height) && intval($svgattrs->height) > 0) {
+                    $height = intval($svgattrs->height);
+                } else {
+                    // Default width.
+                    $height = 600;
+                }
+            }
+
+            $image = [
+                    'width' => $width,
+                    'height' => $height,
+                    'mimetype' => $imagemimetype,
+            ];
+        }
+
+        if (empty($image['width']) or empty($image['height']) or empty($image['mimetype'])) {
+            // GD can not parse it, sorry.
+            return false;
+        }
+        return $image;
+    }
+
+    /**
+     * Serve file content using X-Sendfile header.
+     * Please make sure that all headers are already sent and the all
+     * access control checks passed.
+     *
+     * This alternate method to xsendfile() allows an alternate file system
+     * to use the full file metadata and avoid extra lookups.
+     *
+     * @param stored_file $file The file to send
+     * @return bool success
+     */
+    public function xsendfile_file(stored_file $file): bool {
+        return $this->xsendfile($file->get_contenthash());
+    }
+
+    /**
+     * Serve file content using X-Sendfile header.
+     * Please make sure that all headers are already sent and the all
+     * access control checks passed.
+     *
+     * @param string $contenthash The content hash of the file to be served
+     * @return bool success
+     */
+    public function xsendfile($contenthash) {
+        global $CFG;
+        require_once($CFG->libdir . "/xsendfilelib.php");
+
+        return xsendfile($this->get_remote_path_from_hash($contenthash));
+    }
+
+    /**
+     * Returns true if filesystem is configured to support xsendfile.
+     *
+     * @return bool
+     */
+    public function supports_xsendfile() {
+        global $CFG;
+        return !empty($CFG->xsendfile);
+    }
+
+    /**
+     * Validate that the content hash matches the content hash of the file on disk.
+     *
+     * @param string $contenthash The current content hash to validate
+     * @param string $pathname The path to the file on disk
+     * @return array The content hash (it might change) and file size
+     */
+    protected function validate_hash_and_file_size($contenthash, $pathname) {
+        global $CFG;
+
+        if (!is_readable($pathname)) {
+            throw new file_exception('storedfilecannotread', '', $pathname);
+        }
+
+        $filesize = filesize($pathname);
+        if ($filesize === false) {
+            throw new file_exception('storedfilecannotread', '', $pathname);
+        }
+
+        if (is_null($contenthash)) {
+            $contenthash = file_storage::hash_from_path($pathname);
+        } else if ($CFG->debugdeveloper) {
+            $filehash = file_storage::hash_from_path($pathname);
+            if ($filehash === false) {
+                throw new file_exception('storedfilecannotread', '', $pathname);
+            }
+            if ($filehash !== $contenthash) {
+                // Hopefully this never happens, if yes we need to fix calling code.
+                debugging("Invalid contenthash submitted for file $pathname", DEBUG_DEVELOPER);
+                $contenthash = $filehash;
+            }
+        }
+        if ($contenthash === false) {
+            throw new file_exception('storedfilecannotread', '', $pathname);
+        }
+
+        if ($filesize > 0 and $contenthash === file_storage::hash_from_string('')) {
+            // Did the file change or is file_storage::hash_from_path() borked for this file?
+            clearstatcache();
+            $contenthash = file_storage::hash_from_path($pathname);
+            $filesize    = filesize($pathname);
+
+            if ($contenthash === false or $filesize === false) {
+                throw new file_exception('storedfilecannotread', '', $pathname);
+            }
+            if ($filesize > 0 and $contenthash === file_storage::hash_from_string('')) {
+                // This is very weird...
+                throw new file_exception('storedfilecannotread', '', $pathname);
+            }
+        }
+
+        return [$contenthash, $filesize];
+    }
+
+    /**
+     * Add the supplied file to the file system.
+     *
+     * Note: If overriding this function, it is advisable to store the file
+     * in the path returned by get_local_path_from_hash as there may be
+     * subsequent uses of the file in the same request.
+     *
+     * @param string $pathname Path to file currently on disk
+     * @param string $contenthash SHA1 hash of content if known (performance only)
+     * @return array (contenthash, filesize, newfile)
+     */
+    abstract public function add_file_from_path($pathname, $contenthash = null);
+
+    /**
+     * Add a file with the supplied content to the file system.
+     *
+     * Note: If overriding this function, it is advisable to store the file
+     * in the path returned by get_local_path_from_hash as there may be
+     * subsequent uses of the file in the same request.
+     *
+     * @param string $content file content - binary string
+     * @return array (contenthash, filesize, newfile)
+     */
+    abstract public function add_file_from_string($content);
+
+    /**
+     * Returns file handle - read only mode, no writing allowed into pool files!
+     *
+     * When you want to modify a file, create a new file and delete the old one.
+     *
+     * @param stored_file $file The file to retrieve a handle for
+     * @param int $type Type of file handle (FILE_HANDLE_xx constant)
+     * @return resource file handle
+     */
+    public function get_content_file_handle(stored_file $file, $type = stored_file::FILE_HANDLE_FOPEN) {
+        if ($type === stored_file::FILE_HANDLE_GZOPEN) {
+            // Local file required for gzopen.
+            $path = $this->get_local_path_from_storedfile($file, true);
+        } else {
+            $path = $this->get_remote_path_from_storedfile($file);
+        }
+
+        return self::get_file_handle_for_path($path, $type);
+    }
+
+    /**
+     * Return a file handle for the specified path.
+     *
+     * This abstraction should be used when overriding get_content_file_handle in a new file system.
+     *
+     * @param string $path The path to the file. This shoudl be any type of path that fopen and gzopen accept.
+     * @param int $type Type of file handle (FILE_HANDLE_xx constant)
+     * @return resource
+     * @throws coding_exception When an unexpected type of file handle is requested
+     */
+    protected static function get_file_handle_for_path($path, $type = stored_file::FILE_HANDLE_FOPEN) {
+        switch ($type) {
+            case stored_file::FILE_HANDLE_FOPEN:
+                // Binary reading.
+                return fopen($path, 'rb');
+            case stored_file::FILE_HANDLE_GZOPEN:
+                // Binary reading of file in gz format.
+                return gzopen($path, 'rb');
+            default:
+                throw new coding_exception('Unexpected file handle type');
+        }
+    }
+
+    /**
+     * Get a PSR7 Stream for the specified file which implements the PSR Message StreamInterface.
+     *
+     * @param stored_file $file
+     * @return StreamInterface
+     */
+    public function get_psr_stream(stored_file $file): StreamInterface {
+        return \GuzzleHttp\Psr7\Utils::streamFor($this->get_content_file_handle($file));
+    }
+
+    /**
+     * Retrieve the mime information for the specified stored file.
+     *
+     * @param string $contenthash
+     * @param string $filename
+     * @return string The MIME type.
+     */
+    public function mimetype_from_hash($contenthash, $filename) {
+        $pathname = $this->get_local_path_from_hash($contenthash);
+        $mimetype = file_storage::mimetype($pathname, $filename);
+
+        if ($mimetype === 'document/unknown' && !$this->is_file_readable_locally_by_hash($contenthash)) {
+            // The type is unknown, but the full checks weren't completed because the file isn't locally available.
+            // Ensure we have a local copy and try again.
+            $pathname = $this->get_local_path_from_hash($contenthash, true);
+            $mimetype = file_storage::mimetype_from_file($pathname);
+        }
+
+        return $mimetype;
+    }
+
+    /**
+     * Retrieve the mime information for the specified stored file.
+     *
+     * @param stored_file $file The stored file to retrieve mime information for
+     * @return string The MIME type.
+     */
+    public function mimetype_from_storedfile($file) {
+        if (!$file->get_filesize()) {
+            // Files with an empty filesize are treated as directories and have no mimetype.
+            return null;
+        }
+        return $this->mimetype_from_hash($file->get_contenthash(), $file->get_filename());
+    }
+
+    /**
+     * Run any periodic tasks which must be performed.
+     */
+    public function cron() {
+    }
+}

lib/filestorage/file_system_filedir.php

@@ -0,0 +1,536 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Core file system class definition.
+ *
+ * @package   core_files
+ * @copyright 2017 Andrew Nicols <andrew@nicols.co.uk>
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * File system class used for low level access to real files in filedir.
+ *
+ * @package   core_files
+ * @category  files
+ * @copyright 2017 Andrew Nicols <andrew@nicols.co.uk>
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class file_system_filedir extends file_system {
+
+    /**
+     * @var string The path to the local copy of the filedir.
+     */
+    protected $filedir = null;
+
+    /**
+     * @var string The path to the trashdir.
+     */
+    protected $trashdir = null;
+
+    /**
+     * @var string Default directory permissions for new dirs.
+     */
+    protected $dirpermissions = null;
+
+    /**
+     * @var string Default file permissions for new files.
+     */
+    protected $filepermissions = null;
+
+
+    /**
+     * Perform any custom setup for this type of file_system.
+     */
+    public function __construct() {
+        global $CFG;
+
+        if (isset($CFG->filedir)) {
+            $this->filedir = $CFG->filedir;
+        } else {
+            $this->filedir = $CFG->dataroot.'/filedir';
+        }
+
+        if (isset($CFG->trashdir)) {
+            $this->trashdir = $CFG->trashdir;
+        } else {
+            $this->trashdir = $CFG->dataroot.'/trashdir';
+        }
+
+        $this->dirpermissions = $CFG->directorypermissions;
+        $this->filepermissions = $CFG->filepermissions;
+
+        // Make sure the file pool directory exists.
+        if (!is_dir($this->filedir)) {
+            if (!mkdir($this->filedir, $this->dirpermissions, true)) {
+                // Permission trouble.
+                throw new file_exception('storedfilecannotcreatefiledirs');
+            }
+
+            // Place warning file in file pool root.
+            if (!file_exists($this->filedir.'/warning.txt')) {
+                file_put_contents($this->filedir.'/warning.txt',
+                        'This directory contains the content of uploaded files and is controlled by Moodle code. ' .
+                        'Do not manually move, change or rename any of the files and subdirectories here.');
+                chmod($this->filedir . '/warning.txt', $this->filepermissions);
+            }
+        }
+
+        // Make sure the trashdir directory exists too.
+        if (!is_dir($this->trashdir)) {
+            if (!mkdir($this->trashdir, $this->dirpermissions, true)) {
+                // Permission trouble.
+                throw new file_exception('storedfilecannotcreatefiledirs');
+            }
+        }
+    }
+
+    /**
+     * Get the full path for the specified hash, including the path to the filedir.
+     *
+     * @param string $contenthash The content hash
+     * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
+     * @return string The full path to the content file
+     */
+    protected function get_local_path_from_hash($contenthash, $fetchifnotfound = false) {
+        return $this->get_fulldir_from_hash($contenthash) . '/' .$contenthash;
+    }
+
+    /**
+     * Get a remote filepath for the specified stored file.
+     *
+     * @param stored_file $file The file to fetch the path for
+     * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
+     * @return string The full path to the content file
+     */
+    public function get_local_path_from_storedfile(stored_file $file, $fetchifnotfound = false) {
+        $filepath = $this->get_local_path_from_hash($file->get_contenthash(), $fetchifnotfound);
+
+        // Try content recovery.
+        if ($fetchifnotfound && !is_readable($filepath)) {
+            $this->recover_file($file);
+        }
+
+        return $filepath;
+    }
+
+    /**
+     * Get a remote filepath for the specified stored file.
+     *
+     * @param stored_file $file The file to serve.
+     * @return string full path to pool file with file content
+     */
+    public function get_remote_path_from_storedfile(stored_file $file) {
+        return $this->get_local_path_from_storedfile($file, false);
+    }
+
+    /**
+     * Get the full path for the specified hash, including the path to the filedir.
+     *
+     * @param string $contenthash The content hash
+     * @return string The full path to the content file
+     */
+    protected function get_remote_path_from_hash($contenthash) {
+        return $this->get_local_path_from_hash($contenthash, false);
+    }
+
+    /**
+     * Get the full directory to the stored file, including the path to the
+     * filedir, and the directory which the file is actually in.
+     *
+     * Note: This function does not ensure that the file is present on disk.
+     *
+     * @param stored_file $file The file to fetch details for.
+     * @return string The full path to the content directory
+     */
+    protected function get_fulldir_from_storedfile(stored_file $file) {
+        return $this->get_fulldir_from_hash($file->get_contenthash());
+    }
+
+    /**
+     * Get the full directory to the stored file, including the path to the
+     * filedir, and the directory which the file is actually in.
+     *
+     * @param string $contenthash The content hash
+     * @return string The full path to the content directory
+     */
+    protected function get_fulldir_from_hash($contenthash) {
+        return $this->filedir . '/' . $this->get_contentdir_from_hash($contenthash);
+    }
+
+    /**
+     * Get the content directory for the specified content hash.
+     * This is the directory that the file will be in, but without the
+     * fulldir.
+     *
+     * @param string $contenthash The content hash
+     * @return string The directory within filedir
+     */
+    protected function get_contentdir_from_hash($contenthash) {
+        $l1 = $contenthash[0] . $contenthash[1];
+        $l2 = $contenthash[2] . $contenthash[3];
+        return "$l1/$l2";
+    }
+
+    /**
+     * Get the content path for the specified content hash within filedir.
+     *
+     * This does not include the filedir, and is often used by file systems
+     * as the object key for storage and retrieval.
+     *
+     * @param string $contenthash The content hash
+     * @return string The filepath within filedir
+     */
+    protected function get_contentpath_from_hash($contenthash) {
+        return $this->get_contentdir_from_hash($contenthash) . '/' . $contenthash;
+    }
+
+    /**
+     * Get the full directory for the specified hash in the trash, including the path to the
+     * trashdir, and the directory which the file is actually in.
+     *
+     * @param string $contenthash The content hash
+     * @return string The full path to the trash directory
+     */
+    protected function get_trash_fulldir_from_hash($contenthash) {
+        return $this->trashdir . '/' . $this->get_contentdir_from_hash($contenthash);
+    }
+
+    /**
+     * Get the full path for the specified hash in the trash, including the path to the trashdir.
+     *
+     * @param string $contenthash The content hash
+     * @return string The full path to the trash file
+     */
+    protected function get_trash_fullpath_from_hash($contenthash) {
+        return $this->trashdir . '/' . $this->get_contentpath_from_hash($contenthash);
+    }
+
+    /**
+     * Copy content of file to given pathname.
+     *
+     * @param stored_file $file The file to be copied
+     * @param string $target real path to the new file
+     * @return bool success
+     */
+    public function copy_content_from_storedfile(stored_file $file, $target) {
+        $source = $this->get_local_path_from_storedfile($file, true);
+        return copy($source, $target);
+    }
+
+    /**
+     * Tries to recover missing content of file from trash.
+     *
+     * @param stored_file $file stored_file instance
+     * @return bool success
+     */
+    protected function recover_file(stored_file $file) {
+        $contentfile = $this->get_local_path_from_storedfile($file, false);
+
+        if (file_exists($contentfile)) {
+            // The file already exists on the file system. No need to recover.
+            return true;
+        }
+
+        $contenthash = $file->get_contenthash();
+        $contentdir = $this->get_fulldir_from_storedfile($file);
+        $trashfile = $this->get_trash_fullpath_from_hash($contenthash);
+        $alttrashfile = "{$this->trashdir}/{$contenthash}";
+
+        if (!is_readable($trashfile)) {
+            // The trash file was not found. Check the alternative trash file too just in case.
+            if (!is_readable($alttrashfile)) {
+                return false;
+            }
+            // The alternative trash file in trash root exists.
+            $trashfile = $alttrashfile;
+        }
+
+        if (filesize($trashfile) != $file->get_filesize() or file_storage::hash_from_path($trashfile) != $contenthash) {
+            // The files are different. Leave this one in trash - something seems to be wrong with it.
+            return false;
+        }
+
+        if (!is_dir($contentdir)) {
+            if (!mkdir($contentdir, $this->dirpermissions, true)) {
+                // Unable to create the target directory.
+                return false;
+            }
+        }
+
+        // Perform a rename - these are generally atomic which gives us big
+        // performance wins, especially for large files.
+        return rename($trashfile, $contentfile);
+    }
+
+    /**
+     * Marks pool file as candidate for deleting.
+     *
+     * @param string $contenthash
+     */
+    public function remove_file($contenthash) {
+        if (!self::is_file_removable($contenthash)) {
+            // Don't remove the file - it's still in use.
+            return;
+        }
+
+        if (!$this->is_file_readable_remotely_by_hash($contenthash)) {
+            // The file wasn't found in the first place. Just ignore it.
+            return;
+        }
+
+        $trashpath  = $this->get_trash_fulldir_from_hash($contenthash);
+        $trashfile  = $this->get_trash_fullpath_from_hash($contenthash);
+        $contentfile = $this->get_local_path_from_hash($contenthash, true);
+
+        if (!is_dir($trashpath)) {
+            mkdir($trashpath, $this->dirpermissions, true);
+        }
+
+        if (file_exists($trashfile)) {
+            // A copy of this file is already in the trash.
+            // Remove the old version.
+            unlink($contentfile);
+            return;
+        }
+
+        // Move the contentfile to the trash, and fix permissions as required.
+        rename($contentfile, $trashfile);
+
+        // Fix permissions, only if needed.
+        $currentperms = octdec(substr(decoct(fileperms($trashfile)), -4));
+        if ((int)$this->filepermissions !== $currentperms) {
+            chmod($trashfile, $this->filepermissions);
+        }
+    }
+
+    /**
+     * Cleanup the trash directory.
+     */
+    public function cron() {
+        $this->empty_trash();
+    }
+
+    protected function empty_trash() {
+        fulldelete($this->trashdir);
+        set_config('fileslastcleanup', time());
+    }
+
+    /**
+     * Add the supplied file to the file system.
+     *
+     * Note: If overriding this function, it is advisable to store the file
+     * in the path returned by get_local_path_from_hash as there may be
+     * subsequent uses of the file in the same request.
+     *
+     * @param string $pathname Path to file currently on disk
+     * @param string $contenthash SHA1 hash of content if known (performance only)
+     * @return array (contenthash, filesize, newfile)
+     */
+    public function add_file_from_path($pathname, $contenthash = null) {
+
+        list($contenthash, $filesize) = $this->validate_hash_and_file_size($contenthash, $pathname);
+
+        $hashpath = $this->get_fulldir_from_hash($contenthash);
+        $hashfile = $this->get_local_path_from_hash($contenthash, false);
+
+        $newfile = true;
+
+        $hashsize = self::check_file_exists_and_get_size($hashfile);
+        if ($hashsize !== null) {
+            if ($hashsize === $filesize) {
+                return array($contenthash, $filesize, false);
+            }
+            if (file_storage::hash_from_path($hashfile) === $contenthash) {
+                // Jackpot! We have a hash collision.
+                mkdir("$this->filedir/jackpot/", $this->dirpermissions, true);
+                copy($pathname, "$this->filedir/jackpot/{$contenthash}_1");
+                copy($hashfile, "$this->filedir/jackpot/{$contenthash}_2");
+                throw new file_pool_content_exception($contenthash);
+            }
+            debugging("Replacing invalid content file $contenthash");
+            unlink($hashfile);
+            $newfile = false;
+        }
+
+        if (!is_dir($hashpath)) {
+            if (!mkdir($hashpath, $this->dirpermissions, true)) {
+                // Permission trouble.
+                throw new file_exception('storedfilecannotcreatefiledirs');
+            }
+        }
+
+        // Let's try to prevent some race conditions.
+
+        $prev = ignore_user_abort(true);
+        if (file_exists($hashfile.'.tmp')) {
+            @unlink($hashfile.'.tmp');
+        }
+        if (!copy($pathname, $hashfile.'.tmp')) {
+            // Borked permissions or out of disk space.
+            @unlink($hashfile.'.tmp');
+            ignore_user_abort($prev);
+            throw new file_exception('storedfilecannotcreatefile');
+        }
+        if (file_storage::hash_from_path($hashfile.'.tmp') !== $contenthash) {
+            // Highly unlikely edge case, but this can happen on an NFS volume with no space remaining.
+            @unlink($hashfile.'.tmp');
+            ignore_user_abort($prev);
+            throw new file_exception('storedfilecannotcreatefile');
+        }
+        if (!rename($hashfile.'.tmp', $hashfile)) {
+            // Something very strange went wrong.
+            @unlink($hashfile . '.tmp');
+            // Note, we don't try to clean up $hashfile. Almost certainly, if it exists
+            // (e.g. written by another process?) it will be right, so don't wipe it.
+            ignore_user_abort($prev);
+            throw new file_exception('storedfilecannotcreatefile');
+        }
+        chmod($hashfile, $this->filepermissions); // Fix permissions if needed.
+        if (file_exists($hashfile.'.tmp')) {
+            // Just in case anything fails in a weird way.
+            @unlink($hashfile.'.tmp');
+        }
+        ignore_user_abort($prev);
+
+        return array($contenthash, $filesize, $newfile);
+    }
+
+    /**
+     * Checks if the file exists and gets its size. This function avoids a specific issue with
+     * networked file systems if they incorrectly report the file exists, but then decide it doesn't
+     * as soon as you try to get the file size.
+     *
+     * @param string $hashfile File to check
+     * @return int|null Null if the file does not exist, or the result of filesize(), or -1 if error
+     */
+    protected static function check_file_exists_and_get_size(string $hashfile): ?int {
+        if (!file_exists($hashfile)) {
+            // The file does not exist, return null.
+            return null;
+        }
+
+        // In some networked file systems, it's possible that file_exists will return true when
+        // the file doesn't exist (due to caching), but filesize will then return false because
+        // it doesn't exist.
+        $hashsize = @filesize($hashfile);
+        if ($hashsize !== false) {
+            // We successfully got a file size. Return it.
+            return $hashsize;
+        }
+
+        // If we can't get the filesize, let's check existence again to see if we really
+        // for sure think it exists.
+        clearstatcache();
+        if (!file_exists($hashfile)) {
+            // The file doesn't exist any more, so return null.
+            return null;
+        }
+
+        // It still thinks the file exists, but filesize failed, so we had better return an invalid
+        // value for filesize.
+        return -1;
+    }
+
+    /**
+     * Add a file with the supplied content to the file system.
+     *
+     * Note: If overriding this function, it is advisable to store the file
+     * in the path returned by get_local_path_from_hash as there may be
+     * subsequent uses of the file in the same request.
+     *
+     * @param string $content file content - binary string
+     * @return array (contenthash, filesize, newfile)
+     */
+    public function add_file_from_string($content) {
+        global $CFG;
+
+        $contenthash = file_storage::hash_from_string($content);
+        // Binary length.
+        $filesize = strlen($content ?? '');
+
+        $hashpath = $this->get_fulldir_from_hash($contenthash);
+        $hashfile = $this->get_local_path_from_hash($contenthash, false);
+
+        $newfile = true;
+
+        $hashsize = self::check_file_exists_and_get_size($hashfile);
+        if ($hashsize !== null) {
+            if ($hashsize === $filesize) {
+                return array($contenthash, $filesize, false);
+            }
+            if (file_storage::hash_from_path($hashfile) === $contenthash) {
+                // Jackpot! We have a hash collision.
+                mkdir("$this->filedir/jackpot/", $this->dirpermissions, true);
+                copy($hashfile, "$this->filedir/jackpot/{$contenthash}_1");
+                file_put_contents("$this->filedir/jackpot/{$contenthash}_2", $content);
+                throw new file_pool_content_exception($contenthash);
+            }
+            debugging("Replacing invalid content file $contenthash");
+            unlink($hashfile);
+            $newfile = false;
+        }
+
+        if (!is_dir($hashpath)) {
+            if (!mkdir($hashpath, $this->dirpermissions, true)) {
+                // Permission trouble.
+                throw new file_exception('storedfilecannotcreatefiledirs');
+            }
+        }
+
+        // Hopefully this works around most potential race conditions.
+
+        $prev = ignore_user_abort(true);
+
+        if (!empty($CFG->preventfilelocking)) {
+            $newsize = file_put_contents($hashfile.'.tmp', $content);
+        } else {
+            $newsize = file_put_contents($hashfile.'.tmp', $content, LOCK_EX);
+        }
+
+        if ($newsize === false) {
+            // Borked permissions most likely.
+            ignore_user_abort($prev);
+            throw new file_exception('storedfilecannotcreatefile');
+        }
+        if (filesize($hashfile.'.tmp') !== $filesize) {
+            // Out of disk space?
+            unlink($hashfile.'.tmp');
+            ignore_user_abort($prev);
+            throw new file_exception('storedfilecannotcreatefile');
+        }
+        if (!rename($hashfile.'.tmp', $hashfile)) {
+            // Something very strange went wrong.
+            @unlink($hashfile . '.tmp');
+            // Note, we don't try to clean up $hashfile. Almost certainly, if it exists
+            // (e.g. written by another process?) it will be right, so don't wipe it.
+            ignore_user_abort($prev);
+            throw new file_exception('storedfilecannotcreatefile');
+        }
+        chmod($hashfile, $this->filepermissions); // Fix permissions if needed.
+        if (file_exists($hashfile.'.tmp')) {
+            // Just in case anything fails in a weird way.
+            @unlink($hashfile.'.tmp');
+        }
+        ignore_user_abort($prev);
+
+        return array($contenthash, $filesize, $newfile);
+    }
+
+}

lib/filestorage/mbz_packer.php

@@ -0,0 +1,175 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Implementation of .mbz packer.
+ *
+ * This packer supports .mbz files which can be either .zip or .tar.gz format
+ * internally. A suitable format is chosen depending on system option when
+ * creating new files.
+ *
+ * Internally this packer works by wrapping the existing .zip/.tar.gz packers.
+ *
+ * Backup filenames do not contain non-ASCII characters so packers that do not
+ * support UTF-8 (like the current .tar.gz packer, and possibly external zip
+ * software in some cases if used) can be used by this packer.
+ *
+ * @package core_files
+ * @copyright 2013 The Open University
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+require_once("$CFG->libdir/filestorage/file_packer.php");
+
+/**
+ * Utility class - handles all packing/unpacking of .mbz files.
+ *
+ * @package core_files
+ * @category files
+ * @copyright 2013 The Open University
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class mbz_packer extends file_packer {
+    /**
+     * Archive files and store the result in file storage.
+     *
+     * Any existing file at that location will be overwritten.
+     *
+     * @param array $files array from archive path => pathname or stored_file
+     * @param int $contextid context ID
+     * @param string $component component
+     * @param string $filearea file area
+     * @param int $itemid item ID
+     * @param string $filepath file path
+     * @param string $filename file name
+     * @param int $userid user ID
+     * @param bool $ignoreinvalidfiles true means ignore missing or invalid files, false means abort on any error
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return stored_file|bool false if error stored_file instance if ok
+     * @throws file_exception If file operations fail
+     * @throws coding_exception If any archive paths do not meet the restrictions
+     */
+    public function archive_to_storage(array $files, $contextid,
+            $component, $filearea, $itemid, $filepath, $filename,
+            $userid = null, $ignoreinvalidfiles = true, file_progress $progress = null) {
+        return $this->get_packer_for_archive_operation()->archive_to_storage($files,
+                $contextid, $component, $filearea, $itemid, $filepath, $filename,
+                $userid, $ignoreinvalidfiles, $progress);
+    }
+
+    /**
+     * Archive files and store the result in an OS file.
+     *
+     * @param array $files array from archive path => pathname or stored_file
+     * @param string $archivefile path to target zip file
+     * @param bool $ignoreinvalidfiles true means ignore missing or invalid files, false means abort on any error
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return bool true if file created, false if not
+     * @throws coding_exception If any archive paths do not meet the restrictions
+     */
+    public function archive_to_pathname(array $files, $archivefile,
+            $ignoreinvalidfiles=true, file_progress $progress = null) {
+        return $this->get_packer_for_archive_operation()->archive_to_pathname($files,
+                $archivefile, $ignoreinvalidfiles, $progress);
+    }
+
+    /**
+     * Extract file to given file path (real OS filesystem), existing files are overwritten.
+     *
+     * @param stored_file|string $archivefile full pathname of zip file or stored_file instance
+     * @param string $pathname target directory
+     * @param array $onlyfiles only extract files present in the array
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @param bool $returnbool Whether to return a basic true/false indicating error state, or full per-file error
+     * details.
+     * @return array list of processed files (name=>true)
+     * @throws moodle_exception If error
+     */
+    public function extract_to_pathname($archivefile, $pathname,
+            array $onlyfiles = null, file_progress $progress = null, $returnbool = false) {
+        return $this->get_packer_for_read_operation($archivefile)->extract_to_pathname(
+                $archivefile, $pathname, $onlyfiles, $progress, $returnbool);
+    }
+
+    /**
+     * Extract file to given file path (real OS filesystem), existing files are overwritten.
+     *
+     * @param string|stored_file $archivefile full pathname of zip file or stored_file instance
+     * @param int $contextid context ID
+     * @param string $component component
+     * @param string $filearea file area
+     * @param int $itemid item ID
+     * @param string $pathbase file path
+     * @param int $userid user ID
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return array list of processed files (name=>true)
+     * @throws moodle_exception If error
+     */
+    public function extract_to_storage($archivefile, $contextid,
+            $component, $filearea, $itemid, $pathbase, $userid = null,
+            file_progress $progress = null) {
+        return $this->get_packer_for_read_operation($archivefile)->extract_to_storage(
+                $archivefile, $contextid, $component, $filearea, $itemid, $pathbase,
+                $userid, $progress);
+    }
+
+    /**
+     * Returns array of info about all files in archive.
+     *
+     * @param string|stored_file $archivefile
+     * @return array of file infos
+     */
+    public function list_files($archivefile) {
+        return $this->get_packer_for_read_operation($archivefile)->list_files($archivefile);
+    }
+
+    /**
+     * Selects appropriate packer for new archive depending on system option
+     * and whether required extension is available.
+     *
+     * @return file_packer Suitable packer
+     */
+    protected function get_packer_for_archive_operation() {
+        global $CFG;
+        require_once($CFG->dirroot . '/lib/filestorage/tgz_packer.php');
+
+        if (!empty($CFG->usezipbackups)) {
+            // Allow forced use of zip backups.
+            return get_file_packer('application/zip');
+        } else {
+            return get_file_packer('application/x-gzip');
+        }
+    }
+
+    /**
+     * Selects appropriate packer for existing archive depending on file contents.
+     *
+     * @param string|stored_file $archivefile full pathname of zip file or stored_file instance
+     * @return file_packer Suitable packer
+     */
+    protected function get_packer_for_read_operation($archivefile) {
+        global $CFG;
+        require_once($CFG->dirroot . '/lib/filestorage/tgz_packer.php');
+
+        if (tgz_packer::is_tgz_file($archivefile)) {
+            return get_file_packer('application/x-gzip');
+        } else {
+            return get_file_packer('application/zip');
+        }
+    }
+}

lib/filestorage/tests/fixtures/test.txt

@@ -0,0 +1 @@
+test

lib/filestorage/tests/fixtures/testimage.svg

@@ -0,0 +1,14 @@
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd" [
+        <!ENTITY ns_flows "http://ns.adobe.com/Flows/1.0/">
+        ]>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMinYMid meet">
+    <g id='gtop' stroke-width="12" stroke="#000">
+        <g id="svgstar" transform="translate(50,50)">
+            <path id="svgbar" d="M-27-5a7,7,0,1,0,0,10h54a7,7,0,1,0,0-10z"/>
+            <use id='use1' xlink:href="#svgbar" transform="rotate(45)"/>
+            <use id='use2' xlink:href="#svgbar" transform="rotate(90)"/>
+            <use id='use3' xlink:href="#svgbar" transform="rotate(135)"/>
+        </g>
+    </g>
+    <use id="usetop" xlink:href="#svgstar" fill="#FB4"/>
+</svg>

lib/filestorage/tests/fixtures/testimage_basic.svg

@@ -0,0 +1,6 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100" stroke="#000" preserveAspectRatio="xMinYMid meet">
+  <path d="M8,80s-5,8,5,9l78,0s9,0,5-9l-40-71s-4-6-8,0z" stroke-width="2" fill="#fff" fill-rule="evenodd" />
+  <path d="M52,10 L10,85 L93,85z" stroke-width="2" stroke-linejoin="round" fill="#fc0" fill-rule="evenodd"/>
+  <path d="M52,32l0,26" stroke-width="9" stroke-linecap="round" fill-rule="evenodd"/>
+  <circle r="6" cx="52" cy="73"/>
+</svg>

lib/filestorage/tests/fixtures/testimage_error.svg

@@ -0,0 +1,13 @@
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd" [
+        <!ENTITY ns_flows "http://ns.adobe.com/Flows/1.0/">
+        ]>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMinYMid meet">
+    <g id='gtop' stroke-width="12" stroke="#000">
+        <g id="svgstar" transform="translate(50,50)">
+            <path id="svgbar" d="M-27-5a7,7,0,1,0,0,10h54a7,7,0,1,0,0-10z"/>
+            <use id='use1' xlink:href="#svgbar" transform="rotate(45)"/>
+            <use id='use2' xlink:href="#svgbar" transform="rotate(90)"/>
+            <use id='use3' xlink:href="#svgbar" transform="rotate(135)"/>
+    </g>
+    <use id="usetop" xlink:href="#svgstar" fill="#FB4"/>
+</svg>

lib/filestorage/tests/fixtures/testimage_viewbox.svg

@@ -0,0 +1,14 @@
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd" [
+        <!ENTITY ns_flows "http://ns.adobe.com/Flows/1.0/">
+        ]>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 100 100" preserveAspectRatio="xMinYMid meet">
+    <g id='gtop' stroke-width="12" stroke="#000">
+        <g id="svgstar" transform="translate(50,50)">
+            <path id="svgbar" d="M-27-5a7,7,0,1,0,0,10h54a7,7,0,1,0,0-10z"/>
+            <use id='use1' xlink:href="#svgbar" transform="rotate(45)"/>
+            <use id='use2' xlink:href="#svgbar" transform="rotate(90)"/>
+            <use id='use3' xlink:href="#svgbar" transform="rotate(135)"/>
+        </g>
+    </g>
+    <use id="usetop" xlink:href="#svgstar" fill="#FB4"/>
+</svg>

lib/filestorage/tests/fixtures/testimage_width_height.svg

@@ -0,0 +1,14 @@
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd" [
+        <!ENTITY ns_flows "http://ns.adobe.com/Flows/1.0/">
+        ]>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="100px" height="100px" preserveAspectRatio="xMinYMid meet">
+    <g id='gtop' stroke-width="12" stroke="#000">
+        <g id="svgstar" transform="translate(50,50)">
+            <path id="svgbar" d="M-27-5a7,7,0,1,0,0,10h54a7,7,0,1,0,0-10z"/>
+            <use id='use1' xlink:href="#svgbar" transform="rotate(45)"/>
+            <use id='use2' xlink:href="#svgbar" transform="rotate(90)"/>
+            <use id='use3' xlink:href="#svgbar" transform="rotate(135)"/>
+        </g>
+    </g>
+    <use id="usetop" xlink:href="#svgstar" fill="#FB4"/>
+</svg>

lib/filestorage/tests/fixtures/zip_create_test_file.php

@@ -0,0 +1,55 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * This debug script is used during zip support development.
+ *
+ * @package    core_files
+ * @copyright  2012 Petr Skoda
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+define('CLI_SCRIPT', true);
+
+require(__DIR__.'/../../../../config.php');
+require_once($CFG->libdir.'/clilib.php');
+
+$help =
+    "Create sample zip file for testing
+Example:
+  \$php zip_create_test_file.php test.zip
+";
+
+if (count($_SERVER['argv']) != 2 or file_exists($_SERVER['argv'][1])) {
+    echo $help;
+    exit(0);
+}
+
+$archive = $_SERVER['argv'][1];
+
+$packer = get_file_packer('application/zip');
+
+$file = __DIR__.'/test.txt';
+$files = array(
+    'test.test' => $file,
+    'testíček.txt' => $file,
+    'Prüfung.txt' => $file,
+    '测试.txt' => $file,
+    '試験.txt' => $file,
+    'Žluťoučký/Koníček.txt' => $file,
+);
+
+$packer->archive_to_pathname($files, $archive);

lib/filestorage/tests/fixtures/zip_info.php

@@ -0,0 +1,265 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * This debug script is used during zip support development ONLY.
+ *
+ * @package    core_files
+ * @copyright  2012 Petr Skoda
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+define('CLI_SCRIPT', true);
+
+require(__DIR__.'/../../../../config.php');
+require_once("$CFG->libdir/clilib.php");
+require_once("$CFG->libdir/filestorage/zip_packer.php");
+
+if (count($_SERVER['argv']) != 2 or !file_exists($_SERVER['argv'][1])) {
+    cli_error("This script expects zip file name as the only parameter");
+}
+
+$archive = $_SERVER['argv'][1];
+
+// Note: the ZIP structure is described at http://www.pkware.com/documents/casestudies/APPNOTE.TXT
+if (!$filesize = filesize($archive) or !$fp = fopen($archive, 'rb+')) {
+    cli_error("Can not open ZIP archive: $archive");
+}
+
+if ($filesize == 22) {
+    $info = unpack('Vsig', fread($fp, 4));
+    fclose($fp);
+    if ($info['sig'] == 0x6054b50) {
+        cli_error("This ZIP archive is empty: $archive");
+    } else {
+        cli_error("This is not a ZIP archive: $archive");
+    }
+}
+
+fseek($fp, 0);
+$info = unpack('Vsig', fread($fp, 4));
+if ($info['sig'] !== 0x04034b50) {
+    fclose($fp);
+    cli_error("This is not a ZIP archive: $archive");
+}
+
+// Find end of central directory record.
+$centralend = zip_archive::zip_get_central_end($fp, $filesize);
+if ($centralend === false) {
+    cli_error("This is not a ZIP archive: $archive");
+}
+
+if ($centralend['disk'] !== 0 or $centralend['disk_start'] !== 0) {
+    cli_error("Multi-disk archives are not supported: $archive");
+}
+
+if ($centralend['offset'] === 0xFFFFFFFF) {
+    cli_error("ZIP64 archives are not supported: $archive");
+}
+
+fseek($fp, $centralend['offset']);
+$data = fread($fp, $centralend['size']);
+$pos = 0;
+$files = array();
+for($i=0; $i<$centralend['entries']; $i++) {
+    $file = zip_archive::zip_parse_file_header($data, $centralend, $pos);
+    if ($file === false) {
+        cli_error('Invalid Zip file header structure: '.$archive);
+    }
+
+    // Read local file header.
+    fseek($fp, $file['local_offset']);
+    $localfile = unpack('Vsig/vversion_req/vgeneral/vmethod/Vmodified/Vcrc/Vsize_compressed/Vsize/vname_length/vextra_length', fread($fp, 30));
+    if ($localfile['sig'] !== 0x04034b50) {
+        // Borked file!
+        $file['error'] = 'Invalid local file signature';
+        $files[] = $file;
+        continue;
+    }
+    if ($localfile['name_length']) {
+        $localfile['name'] = fread($fp, $localfile['name_length']);
+    } else {
+        $localfile['name'] = '';
+    }
+    $localfile['extra'] = array();
+    $localfile['extra_data'] = '';
+    if ($localfile['extra_length']) {
+        $extradata = fread($fp, $localfile['extra_length']);
+        $localfile['extra_data'] = $extradata;
+        while (strlen($extradata) > 4) {
+            $extra = unpack('vid/vsize', substr($extradata, 0, 4));
+            $extra['data'] = substr($extradata, 4, $extra['size']);
+            $extradata = substr($extradata, 4+$extra['size']);
+            $localfile['extra'][] = $extra;
+        }
+    }
+
+    $file['local'] = $localfile;
+    $files[] = $file;
+}
+
+echo "Archive:         $archive\n";
+echo "Number of files: {$centralend['entries']}\n";
+echo "Archive comment: \"{$centralend['comment']}\" ({$centralend['comment_length']} bytes)\n";
+foreach ($files as $i=>$file) {
+    echo "======== File ".($i+1)." ==============================================\n";
+    echo "  Name:           ".zip_print_name($file['name'])."\n";
+    if ($file['comment'] !== '') {
+        echo "  Comment:        \"{$file['comment']}\" ({$file['comment_length']} bytes)\n";
+    }
+    echo "  Version:        0x".str_pad(dechex($file['version']), 4, '0', STR_PAD_LEFT)."\n";
+    echo "  Required:       0x".str_pad(dechex($file['version_req']), 4, '0', STR_PAD_LEFT)."\n";
+    echo "  Method:         ".zip_print_method($file['method'])."\n";
+    echo "  General:        ".zip_print_general($file['general'])."\n";
+    echo "  Modified:       ".userdate(zip_dos2unixtime($file['modified']))."\n";
+    echo "  Size:           ".zip_print_sizes($file['size'], $file['size_compressed'])."\n";
+    echo "  CRC-32:         {$file['crc']}\n";
+    foreach($file['extra'] as $j=>$extra) {
+        echo "  Extra ".($j+1).":        ".zip_print_extra($extra)."\n";
+    }
+    if (!empty($file['local']['error'])) {
+        echo "  Local ERROR:    {$file['local']['error']}\n";
+        continue;
+    }
+    $localfile = $file['local'];
+    if ($localfile['name'] !== $file['name']) {
+        echo "  Local name:     ".zip_print_name($localfile['name'])."\n";
+    }
+    if ($localfile['version_req'] !== $file['version_req']) {
+        echo "  Local required: 0x".str_pad(dechex($localfile['version_req']), 4, '0', STR_PAD_LEFT)."\n";
+    }
+    if ($localfile['method'] !== $file['method']) {
+        echo "  Local method:   ".zip_print_method($localfile['method'])."\n";
+    }
+    if ($localfile['general'] !== $file['general']) {
+        echo "  Local general:  ".zip_print_general($localfile['general'])."\n";
+    }
+    if ($localfile['modified'] !== $file['modified']) {
+        echo "  Local modified: ".userdate(zip_dos2unixtime($localfile['modified']))."\n";
+    }
+    if ($localfile['size'] !== $file['size']) {
+        echo "  Local size:     ".zip_print_sizes($localfile['size'], $localfile['size_compressed'])."\n";
+    }
+    if ($localfile['crc'] !== $file['crc']) {
+        echo "  Local CRC-32:   {$localfile['crc']}\n";
+    }
+    foreach($localfile['extra'] as $j=>$extra) {
+        echo "  Local extra ".($j+1).":  ".zip_print_extra($extra)."\n";
+    }
+}
+
+fclose($fp);
+exit(0);
+
+// === Some useful functions ======================================
+
+function zip_print_name($name) {
+    $size = strlen($name);
+    $crc = crc32($name);
+    return "\"$name\" ($size bytes) - CRC $crc";
+}
+
+function zip_print_method($method) {
+    $desc = '';
+    switch($method) {
+        case 0: $desc = 'Stored'; break;
+        case 1: $desc = 'Shrunk'; break;
+        case 2: $desc = 'Reduced factor 1'; break;
+        case 3: $desc = 'Reduced factor 2'; break;
+        case 4: $desc = 'Reduced factor 3'; break;
+        case 5: $desc = 'Reduced factor 4'; break;
+        case 6: $desc = 'Imploded'; break;
+        case 8: $desc = 'Deflated'; break;
+        case 9: $desc = 'Deflate64'; break;
+        case 10: $desc = 'old IBM TERSE'; break;
+        case 12: $desc = 'BZIP2'; break;
+        case 14: $desc = 'LZMA'; break;
+        case 18: $desc = 'IBM TERSE'; break;
+        case 19: $desc = 'IBM LZ77'; break;
+        case 97: $desc = 'WavPack'; break;
+        case 98: $desc = 'PPMd v1'; break;
+    }
+    if ($desc) {
+        $desc = " ($desc)";
+    }
+    return "0x".str_pad(dechex($method), 4, '0', STR_PAD_LEFT).$desc;
+}
+
+function zip_print_general($general) {
+    $desc = array();
+    if ($general & pow(2, 0)) {
+        $desc[] = 'Encrypted';
+    }
+    if ($general & pow(2, 11)) {
+        $desc[] = 'Unicode name';
+    }
+    if ($desc) {
+        $desc = " (".implode(', ', $desc).")";
+    } else {
+        $desc = '';
+    }
+    return str_pad(decbin($general), 16, '0', STR_PAD_LEFT).$desc;
+}
+
+/**
+ * Convert MS date+time format to unix timestamp:
+ * http://msdn.microsoft.com/en-us/library/windows/desktop/ms724274(v=vs.85).aspx
+ *
+ * Copied from: http://plugins.svn.wordpress.org/wp2epub/trunk/zipcreate/functions.lib.php
+ * author: redmonkey
+ * license: GPL
+ */
+function zip_dos2unixtime($dostime) {
+    $sec = 2 * ($dostime & 0x1f);
+    $min = ($dostime >> 5) & 0x3f;
+    $hrs = ($dostime >> 11) & 0x1f;
+    $day = ($dostime >> 16) & 0x1f;
+    $mon = ($dostime >> 21) & 0x0f;
+    $year = (($dostime >> 25) & 0x7f) + 1980;
+
+    return mktime($hrs, $min, $sec, $mon, $day, $year);
+}
+
+function zip_print_sizes($size, $compressed) {
+    return "$size ==> $compressed bytes";
+}
+
+function zip_print_extra($extra) {
+    $desc = '';
+    $info = "- ".bin2hex($extra['data'])." ({$extra['size']} bytes)";
+    switch($extra['id']) {
+        case 0x0009: $desc = 'OS/2'; break;
+        case 0x000a: $desc = 'NTFS'; break;
+        case 0x000d: $desc = 'UNIX'; break;
+        case 0x5455: $desc = 'Extended timestamp'; break;
+        case 0x5855: $desc = 'Infor-ZIP (original)'; break;
+        case 0x7075:
+            $desc = 'Info-ZIP Unicode path';
+            $data = unpack('cversion/Vcrc', substr($extra['data'], 0, 5));
+            $name = substr($extra['data'], 5);
+            $size = strlen($name);
+            if ($data['version'] === 1) {
+                $info = "- \"$name\" ($size bytes) - CRC {$data['crc']}";
+            }
+            break;
+        case 0x7865: $desc = 'Info-ZIP UNIX (new)'; break;
+        case 0x7875: $desc = 'Info-ZIP UNIX (3rd generation)'; break;
+    }
+    if ($desc) {
+        $desc = " ($desc)";
+    }
+    return "0x".str_pad(dechex($extra['id']), 4, '0', STR_PAD_LEFT)."$desc $info";
+}

lib/filestorage/tests/mbz_packer_test.php

@@ -0,0 +1,142 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace core;
+
+defined('MOODLE_INTERNAL') || die();
+
+global $CFG;
+require_once($CFG->libdir . '/filestorage/file_progress.php');
+
+/**
+ * Unit tests for /lib/filestorage/mbz_packer.php.
+ *
+ * @package core
+ * @copyright 2013 The Open University
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class mbz_packer_test extends \advanced_testcase {
+
+    public function test_archive_with_both_options(): void {
+        global $CFG;
+        $this->resetAfterTest();
+
+        // Get backup packer.
+        $packer = get_file_packer('application/vnd.moodle.backup');
+        require_once($CFG->dirroot . '/lib/filestorage/tgz_packer.php');
+
+        // Set up basic archive contents.
+        $files = array('1.txt' => array('frog'));
+
+        // Create 2 archives (each with one file in) in zip mode.
+        $CFG->usezipbackups = true;
+        $filefalse = $CFG->tempdir . '/false.mbz';
+        $this->assertNotEmpty($packer->archive_to_pathname($files, $filefalse));
+        $context = \context_system::instance();
+        $this->assertNotEmpty($storagefalse = $packer->archive_to_storage(
+                $files, $context->id, 'phpunit', 'data', 0, '/', 'false.mbz'));
+
+        // Create 2 archives in tgz mode.
+        $CFG->usezipbackups = false;
+        $filetrue = $CFG->tempdir . '/true.mbz';
+        $this->assertNotEmpty($packer->archive_to_pathname($files, $filetrue));
+        $context = \context_system::instance();
+        $this->assertNotEmpty($storagetrue = $packer->archive_to_storage(
+                $files, $context->id, 'phpunit', 'data', 0, '/', 'true.mbz'));
+
+        // Check the sizes are different (indicating different formats).
+        $this->assertNotEquals(filesize($filefalse), filesize($filetrue));
+        $this->assertNotEquals($storagefalse->get_filesize(), $storagetrue->get_filesize());
+
+        // Extract files into storage and into filesystem from both formats.
+
+        // Extract to path (zip).
+        $packer->extract_to_pathname($filefalse, $CFG->tempdir);
+        $onefile = $CFG->tempdir . '/1.txt';
+        $this->assertEquals('frog', file_get_contents($onefile));
+        unlink($onefile);
+
+        // Extract to path (tgz).
+        $packer->extract_to_pathname($filetrue, $CFG->tempdir);
+        $onefile = $CFG->tempdir . '/1.txt';
+        $this->assertEquals('frog', file_get_contents($onefile));
+        unlink($onefile);
+
+        // Extract to storage (zip).
+        $packer->extract_to_storage($storagefalse, $context->id, 'phpunit', 'data', 1, '/');
+        $fs = get_file_storage();
+        $out = $fs->get_file($context->id, 'phpunit', 'data', 1, '/', '1.txt');
+        $this->assertNotEmpty($out);
+        $this->assertEquals('frog', $out->get_content());
+
+        // Extract to storage (tgz).
+        $packer->extract_to_storage($storagetrue, $context->id, 'phpunit', 'data', 2, '/');
+        $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/', '1.txt');
+        $this->assertNotEmpty($out);
+        $this->assertEquals('frog', $out->get_content());
+    }
+
+    public function usezipbackups_provider() {
+        return [
+            'Use zips'  => [true],
+            'Use tgz'   => [false],
+        ];
+    }
+
+    /**
+     * @dataProvider usezipbackups_provider
+     */
+    public function test_extract_to_pathname_returnvalue_successful($usezipbackups): void {
+        global $CFG;
+        $this->resetAfterTest();
+
+        $packer = get_file_packer('application/vnd.moodle.backup');
+
+        // Set up basic archive contents.
+        $files = array('1.txt' => array('frog'));
+
+        // Create 2 archives (each with one file in) in zip mode.
+        $CFG->usezipbackups = $usezipbackups;
+
+        $mbzfile = make_request_directory() . '/file.mbz';
+        $packer->archive_to_pathname($files, $mbzfile);
+
+        $target = make_request_directory();
+        $result = $packer->extract_to_pathname($mbzfile, $target, null, null, true);
+        $this->assertTrue($result);
+    }
+
+    /**
+     * @dataProvider usezipbackups_provider
+     */
+    public function test_extract_to_pathname_returnvalue_failure($usezipbackups): void {
+        global $CFG;
+        $this->resetAfterTest();
+
+        $packer = get_file_packer('application/vnd.moodle.backup');
+
+        // Create 2 archives (each with one file in) in zip mode.
+        $CFG->usezipbackups = $usezipbackups;
+
+        $mbzfile = make_request_directory() . '/file.mbz';
+        file_put_contents($mbzfile, 'Content');
+
+        $target = make_request_directory();
+        $result = $packer->extract_to_pathname($mbzfile, $target, null, null, true);
+        $this->assertDebuggingCalledCount(1);
+        $this->assertFalse($result);
+    }
+}

lib/filestorage/tests/stored_file_test.php

@@ -0,0 +1,124 @@
+<?php
+// This file is part of Moodle - https://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace core;
+
+use advanced_testcase;
+use context_system;
+
+/**
+ * Unit tests for lib/filestorage/stored_file.php.
+ *
+ * @package    core_files
+ * @category   test
+ * @covers     \stored_file
+ * @copyright  2022 Mikhail Golenkov <mikhailgolenkov@catalyst-au.net>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class stored_file_test extends advanced_testcase {
+
+    /**
+     * Test that the rotate_image() method does not rotate
+     * an image that is not supposed to be rotated.
+     * @covers ::rotate_image()
+     */
+    public function test_rotate_image_does_not_rotate_image(): void {
+        global $CFG;
+        $this->resetAfterTest();
+
+        $filename = 'testimage.jpg';
+        $filepath = $CFG->dirroot . '/lib/filestorage/tests/fixtures/' . $filename;
+        $filerecord = [
+            'contextid' => context_system::instance()->id,
+            'component' => 'core',
+            'filearea'  => 'unittest',
+            'itemid'    => 0,
+            'filepath'  => '/',
+            'filename'  => $filename,
+        ];
+        $fs = get_file_storage();
+        $storedfile = $fs->create_file_from_pathname($filerecord, $filepath);
+
+        $result = $storedfile->rotate_image();
+        $this->assertIsArray($result);
+        $this->assertCount(2, $result);
+        $this->assertFalse($result[0]);
+        $this->assertFalse($result[1]);
+    }
+
+    /**
+     * Test that the rotate_image() method rotates an image
+     * that is supposed to be rotated.
+     * @covers ::rotate_image()
+     */
+    public function test_rotate_image_rotates_image(): void {
+        global $CFG;
+        $this->resetAfterTest();
+
+        // This image was manually rotated to be upside down. Also, Orientation, ExifImageWidth
+        // and ExifImageLength EXIF tags were written into its metadata.
+        // This is needed to make sure that this image will be rotated by stored_file::rotate_image()
+        // and stored as a new rotated file.
+        $filename = 'testimage_rotated.jpg';
+        $filepath = $CFG->dirroot . '/lib/filestorage/tests/fixtures/' . $filename;
+        $filerecord = [
+            'contextid' => context_system::instance()->id,
+            'component' => 'core',
+            'filearea'  => 'unittest',
+            'itemid'    => 0,
+            'filepath'  => '/',
+            'filename'  => $filename,
+        ];
+        $fs = get_file_storage();
+        $storedfile = $fs->create_file_from_pathname($filerecord, $filepath);
+
+        list ($rotateddata, $size) = $storedfile->rotate_image();
+        $this->assertNotFalse($rotateddata);
+        $this->assertIsArray($size);
+        $this->assertEquals(1200, $size['width']);
+        $this->assertEquals(297, $size['height']);
+    }
+
+    /**
+     * Ensure that get_content_file_handle returns a valid file handle.
+     *
+     * @covers ::get_psr_stream
+     */
+    public function test_get_psr_stream(): void {
+        global $CFG;
+        $this->resetAfterTest();
+
+        $filename = 'testimage.jpg';
+        $filepath = $CFG->dirroot . '/lib/filestorage/tests/fixtures/' . $filename;
+        $filerecord = [
+            'contextid' => context_system::instance()->id,
+            'component' => 'core',
+            'filearea'  => 'unittest',
+            'itemid'    => 0,
+            'filepath'  => '/',
+            'filename'  => $filename,
+        ];
+        $fs = get_file_storage();
+        $file = $fs->create_file_from_pathname($filerecord, $filepath);
+
+        $stream = $file->get_psr_stream();
+        $this->assertInstanceOf(\Psr\Http\Message\StreamInterface::class, $stream);
+        $this->assertEquals(file_get_contents($filepath), $stream->getContents());
+        $this->assertFalse($stream->isWritable());
+        $stream->close();
+    }
+
+}

lib/filestorage/tests/tgz_packer_test.php

@@ -0,0 +1,471 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace core;
+
+use file_progress;
+use tgz_packer;
+
+defined('MOODLE_INTERNAL') || die();
+
+global $CFG;
+require_once($CFG->libdir . '/filestorage/file_progress.php');
+
+/**
+ * Unit tests for /lib/filestorage/tgz_packer.php and tgz_extractor.php.
+ *
+ * @package core
+ * @copyright 2013 The Open University
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class tgz_packer_test extends \advanced_testcase implements file_progress {
+    /**
+     * @var array Progress information passed to the progress reporter
+     */
+    protected $progress;
+
+    /**
+     * Puts contents with specified time.
+     *
+     * @param string $path File path
+     * @param string $contents Contents of file
+     * @param int $mtime Time modified
+     */
+    protected static function file_put_contents_at_time($path, $contents, $mtime) {
+        file_put_contents($path, $contents);
+        touch($path, $mtime);
+    }
+
+    /**
+     * Set up some files to be archived.
+     *
+     * @return array Array listing files of all types
+     */
+    protected function prepare_file_list() {
+        global $CFG;
+        $this->resetAfterTest(true);
+
+        // Make array listing files to archive.
+        $filelist = array();
+
+        // Normal file.
+        self::file_put_contents_at_time($CFG->tempdir . '/file1.txt', 'File 1', 1377993601);
+        $filelist['out1.txt'] = $CFG->tempdir . '/file1.txt';
+
+        // Recursive directory w/ file and directory with file.
+        check_dir_exists($CFG->tempdir . '/dir1/dir2');
+        self::file_put_contents_at_time($CFG->tempdir . '/dir1/file2.txt', 'File 2', 1377993602);
+        self::file_put_contents_at_time($CFG->tempdir . '/dir1/dir2/file3.txt', 'File 3', 1377993603);
+        $filelist['out2'] = $CFG->tempdir . '/dir1';
+
+        // Moodle stored_file.
+        $context = \context_system::instance();
+        $filerecord = array('contextid' => $context->id, 'component' => 'phpunit',
+                'filearea' => 'data', 'itemid' => 0, 'filepath' => '/',
+                'filename' => 'file4.txt', 'timemodified' => 1377993604);
+        $fs = get_file_storage();
+        $sf = $fs->create_file_from_string($filerecord, 'File 4');
+        $filelist['out3.txt'] = $sf;
+
+         // Moodle stored_file directory.
+        $filerecord['itemid'] = 1;
+        $filerecord['filepath'] = '/dir1/';
+        $filerecord['filename'] = 'file5.txt';
+        $filerecord['timemodified'] = 1377993605;
+        $fs->create_file_from_string($filerecord, 'File 5');
+        $filerecord['filepath'] = '/dir1/dir2/';
+        $filerecord['filename'] = 'file6.txt';
+        $filerecord['timemodified'] = 1377993606;
+        $fs->create_file_from_string($filerecord, 'File 6');
+        $filerecord['filepath'] = '/';
+        $filerecord['filename'] = 'excluded.txt';
+        $fs->create_file_from_string($filerecord, 'Excluded');
+        $filelist['out4'] = $fs->get_file($context->id, 'phpunit', 'data', 1, '/dir1/', '.');
+
+        // File stored as raw content.
+        $filelist['out5.txt'] = array('File 7');
+
+        // File where there's just an empty directory.
+        $filelist['out6'] = null;
+
+        return $filelist;
+    }
+
+    /**
+     * Tests getting the item.
+     */
+    public function test_get_packer(): void {
+        $packer = get_file_packer('application/x-gzip');
+        $this->assertInstanceOf('tgz_packer', $packer);
+    }
+
+    /**
+     * Tests basic archive and extract to file paths.
+     */
+    public function test_to_normal_files(): void {
+        global $CFG;
+        $packer = get_file_packer('application/x-gzip');
+
+        // Archive files.
+        $files = $this->prepare_file_list();
+        $archivefile = $CFG->tempdir . '/test.tar.gz';
+        $packer->archive_to_pathname($files, $archivefile);
+
+        // Extract same files.
+        $outdir = $CFG->tempdir . '/out';
+        check_dir_exists($outdir);
+        $result = $packer->extract_to_pathname($archivefile, $outdir);
+
+        // The result array should have file entries + directory entries for
+        // all implicit directories + entry for the explicit directory.
+        $expectedpaths = array('out1.txt', 'out2/', 'out2/dir2/', 'out2/dir2/file3.txt',
+                'out2/file2.txt', 'out3.txt', 'out4/', 'out4/dir2/', 'out4/file5.txt',
+                'out4/dir2/file6.txt', 'out5.txt', 'out6/');
+        sort($expectedpaths);
+        $actualpaths = array_keys($result);
+        sort($actualpaths);
+        $this->assertEquals($expectedpaths, $actualpaths);
+        foreach ($result as $path => $booleantrue) {
+            $this->assertTrue($booleantrue);
+        }
+
+        // Check the files are as expected.
+        $this->assertEquals('File 1', file_get_contents($outdir . '/out1.txt'));
+        $this->assertEquals('File 2', file_get_contents($outdir . '/out2/file2.txt'));
+        $this->assertEquals('File 3', file_get_contents($outdir . '/out2/dir2/file3.txt'));
+        $this->assertEquals('File 4', file_get_contents($outdir . '/out3.txt'));
+        $this->assertEquals('File 5', file_get_contents($outdir . '/out4/file5.txt'));
+        $this->assertEquals('File 6', file_get_contents($outdir . '/out4/dir2/file6.txt'));
+        $this->assertEquals('File 7', file_get_contents($outdir . '/out5.txt'));
+        $this->assertTrue(is_dir($outdir . '/out6'));
+    }
+
+    /**
+     * Tests archive and extract to Moodle file system.
+     */
+    public function test_to_stored_files(): void {
+        global $CFG;
+        $packer = get_file_packer('application/x-gzip');
+
+        // Archive files.
+        $files = $this->prepare_file_list();
+        $archivefile = $CFG->tempdir . '/test.tar.gz';
+        $context = \context_system::instance();
+        $sf = $packer->archive_to_storage($files,
+                $context->id, 'phpunit', 'archive', 1, '/', 'archive.tar.gz');
+        $this->assertInstanceOf('stored_file', $sf);
+
+        // Extract (from storage) to disk.
+        $outdir = $CFG->tempdir . '/out';
+        check_dir_exists($outdir);
+        $packer->extract_to_pathname($sf, $outdir);
+
+        // Check the files are as expected.
+        $this->assertEquals('File 1', file_get_contents($outdir . '/out1.txt'));
+        $this->assertEquals('File 2', file_get_contents($outdir . '/out2/file2.txt'));
+        $this->assertEquals('File 3', file_get_contents($outdir . '/out2/dir2/file3.txt'));
+        $this->assertEquals('File 4', file_get_contents($outdir . '/out3.txt'));
+        $this->assertEquals('File 5', file_get_contents($outdir . '/out4/file5.txt'));
+        $this->assertEquals('File 6', file_get_contents($outdir . '/out4/dir2/file6.txt'));
+        $this->assertEquals('File 7', file_get_contents($outdir . '/out5.txt'));
+        $this->assertTrue(is_dir($outdir . '/out6'));
+
+        // Extract to Moodle storage.
+        $packer->extract_to_storage($sf, $context->id, 'phpunit', 'data', 2, '/out/');
+        $fs = get_file_storage();
+        $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/', 'out1.txt');
+        $this->assertNotEmpty($out);
+        $this->assertEquals('File 1', $out->get_content());
+        $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/out2/', 'file2.txt');
+        $this->assertNotEmpty($out);
+        $this->assertEquals('File 2', $out->get_content());
+        $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/out2/dir2/', 'file3.txt');
+        $this->assertNotEmpty($out);
+        $this->assertEquals('File 3', $out->get_content());
+        $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/', 'out3.txt');
+        $this->assertNotEmpty($out);
+        $this->assertEquals('File 4', $out->get_content());
+        $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/out4/', 'file5.txt');
+        $this->assertNotEmpty($out);
+        $this->assertEquals('File 5', $out->get_content());
+        $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/out4/dir2/', 'file6.txt');
+        $this->assertNotEmpty($out);
+        $this->assertEquals('File 6', $out->get_content());
+        $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/', 'out5.txt');
+        $this->assertNotEmpty($out);
+        $this->assertEquals('File 7', $out->get_content());
+        $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/out6/', '.');
+        $this->assertNotEmpty($out);
+        $this->assertTrue($out->is_directory());
+
+        // These functions are supposed to overwrite existing files; test they
+        // don't give errors when run twice.
+        $sf = $packer->archive_to_storage($files,
+                $context->id, 'phpunit', 'archive', 1, '/', 'archive.tar.gz');
+        $this->assertInstanceOf('stored_file', $sf);
+        $packer->extract_to_storage($sf, $context->id, 'phpunit', 'data', 2, '/out/');
+    }
+
+    /**
+     * Tests extracting with a list of specified files.
+     */
+    public function test_only_specified_files(): void {
+        global $CFG;
+        $packer = get_file_packer('application/x-gzip');
+
+        // Archive files.
+        $files = $this->prepare_file_list();
+        $archivefile = $CFG->tempdir . '/test.tar.gz';
+        $packer->archive_to_pathname($files, $archivefile);
+
+        // Extract same files.
+        $outdir = $CFG->tempdir . '/out';
+        check_dir_exists($outdir);
+        $result = $packer->extract_to_pathname($archivefile, $outdir,
+                array('out3.txt', 'out6/', 'out4/file5.txt'));
+
+        // Check result reporting only includes specified files.
+        $expectedpaths = array('out3.txt', 'out4/file5.txt', 'out6/');
+        sort($expectedpaths);
+        $actualpaths = array_keys($result);
+        sort($actualpaths);
+        $this->assertEquals($expectedpaths, $actualpaths);
+
+        // Check the files are as expected.
+        $this->assertFalse(file_exists($outdir . '/out1.txt'));
+        $this->assertEquals('File 4', file_get_contents($outdir . '/out3.txt'));
+        $this->assertEquals('File 5', file_get_contents($outdir . '/out4/file5.txt'));
+        $this->assertTrue(is_dir($outdir . '/out6'));
+    }
+
+    /**
+     * Tests extracting files returning only a boolean state with success.
+     */
+    public function test_extract_to_pathname_returnvalue_successful(): void {
+        $packer = get_file_packer('application/x-gzip');
+
+        // Prepare files.
+        $files = $this->prepare_file_list();
+        $archivefile = make_request_directory() . '/test.tgz';
+        $packer->archive_to_pathname($files, $archivefile);
+
+        // Extract same files.
+        $outdir = make_request_directory();
+        $result = $packer->extract_to_pathname($archivefile, $outdir, null, null, true);
+
+        $this->assertTrue($result);
+    }
+
+    /**
+     * Tests extracting files returning only a boolean state with failure.
+     */
+    public function test_extract_to_pathname_returnvalue_failure(): void {
+        $packer = get_file_packer('application/x-gzip');
+
+        // Create sample files.
+        $archivefile = make_request_directory() . '/test.tgz';
+        file_put_contents($archivefile, '');
+
+        // Extract same files.
+        $outdir = make_request_directory();
+
+        $result = $packer->extract_to_pathname($archivefile, $outdir, null, null, true);
+
+        $this->assertFalse($result);
+    }
+
+    /**
+     * Tests the progress reporting.
+     */
+    public function test_file_progress(): void {
+        global $CFG;
+
+        // Set up.
+        $filelist = $this->prepare_file_list();
+        $packer = get_file_packer('application/x-gzip');
+        $archive = "$CFG->tempdir/archive.tgz";
+        $context = \context_system::instance();
+
+        // Archive to pathname.
+        $this->progress = array();
+        $result = $packer->archive_to_pathname($filelist, $archive, true, $this);
+        $this->assertTrue($result);
+        // Should send progress at least once per file.
+        $this->assertTrue(count($this->progress) >= count($filelist));
+        // Progress should obey some restrictions.
+        $this->check_progress_toward_max();
+
+        // Archive to storage.
+        $this->progress = array();
+        $archivefile = $packer->archive_to_storage($filelist, $context->id,
+                'phpunit', 'test', 0, '/', 'archive.tgz', null, true, $this);
+        $this->assertInstanceOf('stored_file', $archivefile);
+        $this->assertTrue(count($this->progress) >= count($filelist));
+        $this->check_progress_toward_max();
+
+        // Extract to pathname.
+        $this->progress = array();
+        $target = "$CFG->tempdir/test/";
+        check_dir_exists($target);
+        $result = $packer->extract_to_pathname($archive, $target, null, $this);
+        remove_dir($target);
+        // We only output progress once per block, and this is kind of a small file.
+        $this->assertTrue(count($this->progress) >= 1);
+        $this->check_progress_toward_max();
+
+        // Extract to storage (from storage).
+        $this->progress = array();
+        $result = $packer->extract_to_storage($archivefile, $context->id,
+                'phpunit', 'target', 0, '/', null, $this);
+        $this->assertTrue(count($this->progress) >= 1);
+        $this->check_progress_toward_max();
+
+        // Extract to storage (from path).
+        $this->progress = array();
+        $result = $packer->extract_to_storage($archive, $context->id,
+                'phpunit', 'target', 0, '/', null, $this);
+        $this->assertTrue(count($this->progress) >= 1);
+        $this->check_progress_toward_max();
+
+        // Wipe created disk file.
+        unlink($archive);
+    }
+
+    /**
+     * Tests the list_files function with and without an index file.
+     */
+    public function test_list_files(): void {
+        global $CFG;
+
+        // Set up.
+        $filelist = $this->prepare_file_list();
+        $packer = get_file_packer('application/x-gzip');
+        $archive = "$CFG->tempdir/archive.tgz";
+
+        // Archive with an index (default).
+        $packer = get_file_packer('application/x-gzip');
+        $result = $packer->archive_to_pathname($filelist, $archive, true, $this);
+        $this->assertTrue($result);
+        $hashwith = \file_storage::hash_from_path($archive);
+
+        // List files.
+        $files = $packer->list_files($archive);
+
+        // Check they match expected.
+        $expectedinfo = array(
+            array('out1.txt', 1377993601, false, 6),
+            array('out2/', tgz_packer::DEFAULT_TIMESTAMP, true, 0),
+            array('out2/dir2/', tgz_packer::DEFAULT_TIMESTAMP, true, 0),
+            array('out2/dir2/file3.txt', 1377993603, false, 6),
+            array('out2/file2.txt', 1377993602, false, 6),
+            array('out3.txt', 1377993604, false, 6),
+            array('out4/', tgz_packer::DEFAULT_TIMESTAMP, true, 0),
+            array('out4/dir2/', tgz_packer::DEFAULT_TIMESTAMP, true, 0),
+            array('out4/dir2/file6.txt', 1377993606, false, 6),
+            array('out4/file5.txt', 1377993605, false, 6),
+            array('out5.txt', tgz_packer::DEFAULT_TIMESTAMP, false, 6),
+            array('out6/', tgz_packer::DEFAULT_TIMESTAMP, true, 0),
+        );
+        $this->assertEquals($expectedinfo, self::convert_info_for_assert($files));
+
+        // Archive with no index. Should have same result.
+        $this->progress = array();
+        $packer->set_include_index(false);
+        $result = $packer->archive_to_pathname($filelist, $archive, true, $this);
+        $this->assertTrue($result);
+        $hashwithout = \file_storage::hash_from_path($archive);
+        $files = $packer->list_files($archive);
+        $this->assertEquals($expectedinfo, self::convert_info_for_assert($files));
+
+        // Check it actually is different (does have index in)!
+        $this->assertNotEquals($hashwith, $hashwithout);
+
+        // Put the index back on in case of future tests.
+        $packer->set_include_index(true);
+    }
+
+    /**
+     * Utility function to convert the file info array into a simpler format
+     * for making comparisons.
+     *
+     * @param array $files Array from list_files result
+     */
+    protected static function convert_info_for_assert(array $files) {
+        $actualinfo = array();
+        foreach ($files as $file) {
+            $actualinfo[] = array($file->pathname, $file->mtime, $file->is_directory, $file->size);
+        }
+        usort($actualinfo, function($a, $b) {
+            return strcmp($a[0], $b[0]);
+        });
+        return $actualinfo;
+    }
+
+    public function test_is_tgz_file(): void {
+        global $CFG;
+
+        // Set up.
+        $filelist = $this->prepare_file_list();
+        $packer1 = get_file_packer('application/x-gzip');
+        $packer2 = get_file_packer('application/zip');
+        $archive2 = "$CFG->tempdir/archive.zip";
+
+        // Archive in tgz and zip format.
+        $context = \context_system::instance();
+        $archive1 = $packer1->archive_to_storage($filelist, $context->id,
+                'phpunit', 'test', 0, '/', 'archive.tgz', null, true, $this);
+        $this->assertInstanceOf('stored_file', $archive1);
+        $result = $packer2->archive_to_pathname($filelist, $archive2);
+        $this->assertTrue($result);
+
+        // Use is_tgz_file to detect which is which. First check is from storage,
+        // second check is from filesystem.
+        $this->assertTrue(tgz_packer::is_tgz_file($archive1));
+        $this->assertFalse(tgz_packer::is_tgz_file($archive2));
+    }
+
+    /**
+     * Checks that progress reported is numeric rather than indeterminate,
+     * and follows the progress reporting rules.
+     */
+    protected function check_progress_toward_max() {
+        $lastvalue = -1; $lastmax = -1;
+        foreach ($this->progress as $progressitem) {
+            list($value, $max) = $progressitem;
+            if ($lastmax != -1) {
+                $this->assertEquals($max, $lastmax);
+            } else {
+                $lastmax = $max;
+            }
+            $this->assertTrue(is_integer($value));
+            $this->assertTrue(is_integer($max));
+            $this->assertNotEquals(file_progress::INDETERMINATE, $max);
+            $this->assertTrue($value <= $max);
+            $this->assertTrue($value >= $lastvalue);
+            $lastvalue = $value;
+        }
+    }
+
+    /**
+     * Handles file_progress interface.
+     *
+     * @param int $progress
+     * @param int $max
+     */
+    public function progress($progress = file_progress::INDETERMINATE, $max = file_progress::INDETERMINATE) {
+        $this->progress[] = array($progress, $max);
+    }
+}

lib/filestorage/tests/zip_packer_test.php

@@ -0,0 +1,727 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace core;
+
+use file_archive;
+use file_progress;
+use zip_archive;
+
+defined('MOODLE_INTERNAL') || die();
+
+global $CFG;
+require_once($CFG->libdir . '/filestorage/file_progress.php');
+
+/**
+ * Unit tests for /lib/filestorage/zip_packer.php and zip_archive.php
+ *
+ * @package   core
+ * @category  test
+ * @copyright 2012 Petr Skoda
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class zip_packer_test extends \advanced_testcase implements file_progress {
+    protected $testfile;
+    protected $files;
+
+    /**
+     * @var array Progress information passed to the progress reporter
+     */
+    protected $progress;
+
+    protected function setUp(): void {
+        parent::setUp();
+
+        $this->testfile = __DIR__.'/fixtures/test.txt';
+
+        $fs = get_file_storage();
+        $context = \context_system::instance();
+        if (!$file = $fs->get_file($context->id, 'phpunit', 'data', 0, '/', 'test.txt')) {
+            $file = $fs->create_file_from_pathname(
+                array('contextid'=>$context->id, 'component'=>'phpunit', 'filearea'=>'data', 'itemid'=>0, 'filepath'=>'/', 'filename'=>'test.txt'),
+                $this->testfile);
+        }
+
+        $this->files = array(
+            'test.test' => $this->testfile,
+            'testíček.txt' => $this->testfile,
+            'Prüfung.txt' => $this->testfile,
+            '测试.txt' => $this->testfile,
+            '試験.txt' => $this->testfile,
+            'Žluťoučký/Koníček.txt' => $file,
+        );
+    }
+
+    public function test_get_packer(): void {
+        $this->resetAfterTest(false);
+        $packer = get_file_packer();
+        $this->assertInstanceOf('zip_packer', $packer);
+
+        $packer = get_file_packer('application/zip');
+        $this->assertInstanceOf('zip_packer', $packer);
+    }
+
+    /**
+     * @depends test_get_packer
+     */
+    public function test_list_files(): void {
+        $this->resetAfterTest(false);
+
+        $files = array(
+            __DIR__.'/fixtures/test_moodle_22.zip',
+            __DIR__.'/fixtures/test_moodle.zip',
+            __DIR__.'/fixtures/test_tc_8.zip',
+            __DIR__.'/fixtures/test_7zip_927.zip',
+            __DIR__.'/fixtures/test_winzip_165.zip',
+            __DIR__.'/fixtures/test_winrar_421.zip',
+            __DIR__.'/fixtures/test_thumbsdb.zip',
+        );
+
+        if (function_exists('normalizer_normalize')) {
+            // Unfortunately there is no way to standardise UTF-8 strings without INTL extension.
+            $files[] = __DIR__.'/fixtures/test_infozip_3.zip';
+            $files[] = __DIR__.'/fixtures/test_osx_1074.zip';
+            $files[] = __DIR__.'/fixtures/test_osx_compress.zip';
+        }
+
+        $packer = get_file_packer('application/zip');
+
+        foreach ($files as $archive) {
+            $archivefiles = $packer->list_files($archive);
+            $this->assertTrue(is_array($archivefiles), "Archive not extracted properly: ".basename($archive).' ');
+            $this->assertTrue(count($this->files) === count($archivefiles) or count($this->files) === count($archivefiles) - 1); // Some zippers create empty dirs.
+            foreach ($archivefiles as $file) {
+                if ($file->pathname === 'Žluťoučký/') {
+                    // Some zippers create empty dirs.
+                    continue;
+                }
+                $this->assertArrayHasKey($file->pathname, $this->files, "File $file->pathname not extracted properly: ".basename($archive).' ');
+            }
+        }
+
+        // Windows packer supports only DOS encoding.
+        $archive = __DIR__.'/fixtures/test_win8_de.zip';
+        $archivefiles = $packer->list_files($archive);
+        $this->assertTrue(is_array($archivefiles), "Archive not extracted properly: ".basename($archive).' ');
+        $this->assertEquals(2, count($archivefiles));
+        foreach ($archivefiles as $file) {
+            $this->assertTrue($file->pathname === 'Prüfung.txt' or $file->pathname === 'test.test');
+        }
+
+        $zip_archive = new zip_archive();
+        $zip_archive->open(__DIR__.'/fixtures/test_win8_cz.zip', file_archive::OPEN, 'cp852');
+        $archivefiles = $zip_archive->list_files();
+        $this->assertTrue(is_array($archivefiles), "Archive not extracted properly: ".basename($archive).' ');
+        $this->assertEquals(3, count($archivefiles));
+        foreach ($archivefiles as $file) {
+            $this->assertTrue($file->pathname === 'Žluťoučký/Koníček.txt' or $file->pathname === 'testíček.txt' or $file->pathname === 'test.test');
+        }
+        $zip_archive->close();
+
+        // Empty archive extraction.
+        $archive = __DIR__.'/fixtures/empty.zip';
+        $archivefiles = $packer->list_files($archive);
+        $this->assertSame(array(), $archivefiles);
+    }
+
+    /**
+     * @depends test_list_files
+     */
+    public function test_archive_to_pathname(): void {
+        global $CFG;
+
+        $this->resetAfterTest(false);
+
+        $packer = get_file_packer('application/zip');
+        $archive = "$CFG->tempdir/archive.zip";
+
+        $this->assertFileDoesNotExist($archive);
+        $result = $packer->archive_to_pathname($this->files, $archive);
+        $this->assertTrue($result);
+        $this->assertFileExists($archive);
+
+        $archivefiles = $packer->list_files($archive);
+        $this->assertTrue(is_array($archivefiles));
+        $this->assertEquals(count($this->files), count($archivefiles));
+        foreach ($archivefiles as $file) {
+            $this->assertArrayHasKey($file->pathname, $this->files);
+        }
+
+        // Test invalid files parameter.
+        $archive = "$CFG->tempdir/archive2.zip";
+        $this->assertFileDoesNotExist($archive);
+
+        $this->assertFileDoesNotExist(__DIR__.'/xx/yy/ee.txt');
+        $files = array('xtest.txt'=>__DIR__.'/xx/yy/ee.txt');
+
+        $result = $packer->archive_to_pathname($files, $archive, false);
+        $this->assertFalse($result);
+        $this->assertDebuggingCalled();
+        $this->assertFileDoesNotExist($archive);
+
+        $result = $packer->archive_to_pathname($files, $archive);
+        $this->assertTrue($result);
+        $this->assertFileExists($archive);
+        $this->assertDebuggingCalled();
+        $archivefiles = $packer->list_files($archive);
+        $this->assertSame(array(), $archivefiles);
+        unlink($archive);
+
+        $this->assertFileDoesNotExist(__DIR__.'/xx/yy/ee.txt');
+        $this->assertFileExists(__DIR__.'/fixtures/test.txt');
+        $files = array('xtest.txt'=>__DIR__.'/xx/yy/ee.txt', 'test.txt'=>__DIR__.'/fixtures/test.txt', 'ytest.txt'=>__DIR__.'/xx/yy/yy.txt');
+        $result = $packer->archive_to_pathname($files, $archive);
+        $this->assertTrue($result);
+        $this->assertFileExists($archive);
+        $archivefiles = $packer->list_files($archive);
+        $this->assertCount(1, $archivefiles);
+        $this->assertEquals('test.txt', $archivefiles[0]->pathname);
+        $dms = $this->getDebuggingMessages();
+        $this->assertCount(2, $dms);
+        $this->resetDebugging();
+        unlink($archive);
+    }
+
+    /**
+     * @depends test_archive_to_pathname
+     */
+    public function test_archive_to_storage(): void {
+        $this->resetAfterTest(false);
+
+        $packer = get_file_packer('application/zip');
+        $fs = get_file_storage();
+        $context = \context_system::instance();
+
+        $this->assertFalse($fs->file_exists($context->id, 'phpunit', 'test', 0, '/', 'archive.zip'));
+        $result = $packer->archive_to_storage($this->files, $context->id, 'phpunit', 'test', 0, '/', 'archive.zip');
+        $this->assertInstanceOf('stored_file', $result);
+        $this->assertTrue($fs->file_exists($context->id, 'phpunit', 'test', 0, '/', 'archive.zip'));
+
+        $archivefiles = $result->list_files($packer);
+        $this->assertTrue(is_array($archivefiles));
+        $this->assertEquals(count($this->files), count($archivefiles));
+        foreach ($archivefiles as $file) {
+            $this->assertArrayHasKey($file->pathname, $this->files);
+        }
+    }
+
+    /**
+     * @depends test_archive_to_storage
+     */
+    public function test_extract_to_pathname(): void {
+        global $CFG;
+
+        $this->resetAfterTest(false);
+
+        $packer = get_file_packer('application/zip');
+        $fs = get_file_storage();
+        $context = \context_system::instance();
+
+        $target = "$CFG->tempdir/test/";
+        $testcontent = file_get_contents($this->testfile);
+
+        @mkdir($target, $CFG->directorypermissions);
+        $this->assertTrue(is_dir($target));
+
+        $archive = "$CFG->tempdir/archive.zip";
+        $this->assertFileExists($archive);
+        $result = $packer->extract_to_pathname($archive, $target);
+        $this->assertTrue(is_array($result));
+        $this->assertEquals(count($this->files), count($result));
+        foreach ($this->files as $file => $unused) {
+            $this->assertTrue($result[$file]);
+            $this->assertFileExists($target.$file);
+            $this->assertSame($testcontent, file_get_contents($target.$file));
+        }
+
+        $archive = $fs->get_file($context->id, 'phpunit', 'test', 0, '/', 'archive.zip');
+        $this->assertNotEmpty($archive);
+        $result = $packer->extract_to_pathname($archive, $target);
+        $this->assertTrue(is_array($result));
+        $this->assertEquals(count($this->files), count($result));
+        foreach ($this->files as $file => $unused) {
+            $this->assertTrue($result[$file]);
+            $this->assertFileExists($target.$file);
+            $this->assertSame($testcontent, file_get_contents($target.$file));
+        }
+    }
+
+    /**
+     * Test functionality of {@see zip_packer} for entries with folders ending with dots.
+     *
+     * @link https://bugs.php.net/bug.php?id=77214
+     */
+    public function test_zip_entry_path_having_folder_ending_with_dot(): void {
+        global $CFG;
+
+        $this->resetAfterTest(false);
+
+        $packer = get_file_packer('application/zip');
+        $tmp = make_request_directory();
+        $now = time();
+
+        // Create a test archive containing a folder ending with dot.
+        $zippath = $tmp . '/test_archive.zip';
+        $zipcontents = [
+            'HOW.TO' => ['Just run tests.'],
+            'README.' => ['This is a test ZIP file'],
+            './Current time' => [$now],
+            'Data/sub1./sub2/1221' => ['1221'],
+            'Data/sub1./sub2./Příliš žluťoučký kůň úpěl Ďábelské Ódy.txt' => [''],
+        ];
+
+        if ($CFG->ostype === 'WINDOWS') {
+            // File names cannot end with dots on Windows and trailing dots are replaced with underscore.
+            $filenamemap = [
+                'HOW.TO' => 'HOW.TO',
+                'README.' => 'README_',
+                './Current time' => 'Current time',
+                'Data/sub1./sub2/1221' => 'Data/sub1_/sub2/1221',
+                'Data/sub1./sub2./Příliš žluťoučký kůň úpěl Ďábelské Ódy.txt' =>
+                    'Data/sub1_/sub2_/Příliš žluťoučký kůň úpěl Ďábelské Ódy.txt',
+            ];
+
+        } else {
+            $filenamemap = [
+                'HOW.TO' => 'HOW.TO',
+                'README.' => 'README.',
+                './Current time' => 'Current time',
+                'Data/sub1./sub2/1221' => 'Data/sub1./sub2/1221',
+                'Data/sub1./sub2./Příliš žluťoučký kůň úpěl Ďábelské Ódy.txt' =>
+                    'Data/sub1./sub2./Příliš žluťoučký kůň úpěl Ďábelské Ódy.txt',
+            ];
+        }
+
+        // Check that the archive can be created.
+        $result = $packer->archive_to_pathname($zipcontents, $zippath, false);
+        $this->assertTrue($result);
+
+        // Check list of files.
+        $listfiles = $packer->list_files($zippath);
+        $this->assertEquals(count($zipcontents), count($listfiles));
+
+        foreach ($listfiles as $fileinfo) {
+            $this->assertSame($fileinfo->pathname, $fileinfo->original_pathname);
+            $this->assertArrayHasKey($fileinfo->pathname, $zipcontents);
+        }
+
+        // Check actual extracting.
+        $targetpath = $tmp . '/target';
+        check_dir_exists($targetpath);
+        $result = $packer->extract_to_pathname($zippath, $targetpath, null, null, true);
+
+        $this->assertTrue($result);
+
+        foreach ($zipcontents as $filename => $filecontents) {
+            $filecontents = reset($filecontents);
+            $this->assertTrue(is_readable($targetpath . '/' . $filenamemap[$filename]));
+            $this->assertEquals($filecontents, file_get_contents($targetpath . '/' . $filenamemap[$filename]));
+        }
+    }
+
+    /**
+     * @depends test_archive_to_storage
+     */
+    public function test_extract_to_pathname_onlyfiles(): void {
+        global $CFG;
+
+        $this->resetAfterTest(false);
+
+        $packer = get_file_packer('application/zip');
+        $fs = get_file_storage();
+        $context = \context_system::instance();
+
+        $target = "$CFG->tempdir/onlyfiles/";
+        $testcontent = file_get_contents($this->testfile);
+
+        @mkdir($target, $CFG->directorypermissions);
+        $this->assertTrue(is_dir($target));
+
+        $onlyfiles = array('test', 'test.test', 'Žluťoučký/Koníček.txt', 'Idontexist');
+        $willbeextracted = array_intersect(array_keys($this->files), $onlyfiles);
+        $donotextract = array_diff(array_keys($this->files), $onlyfiles);
+
+        $archive = "$CFG->tempdir/archive.zip";
+        $this->assertFileExists($archive);
+        $result = $packer->extract_to_pathname($archive, $target, $onlyfiles);
+        $this->assertTrue(is_array($result));
+        $this->assertEquals(count($willbeextracted), count($result));
+
+        foreach ($willbeextracted as $file) {
+            $this->assertTrue($result[$file]);
+            $this->assertFileExists($target.$file);
+            $this->assertSame($testcontent, file_get_contents($target.$file));
+        }
+        foreach ($donotextract as $file) {
+            $this->assertFalse(isset($result[$file]));
+            $this->assertFileDoesNotExist($target.$file);
+        }
+
+    }
+
+    /**
+     * @depends test_archive_to_storage
+     */
+    public function test_extract_to_pathname_returnvalue_successful(): void {
+        global $CFG;
+
+        $this->resetAfterTest(false);
+
+        $packer = get_file_packer('application/zip');
+
+        $target = make_request_directory();
+
+        $archive = "$CFG->tempdir/archive.zip";
+        $this->assertFileExists($archive);
+        $result = $packer->extract_to_pathname($archive, $target, null, null, true);
+        $this->assertTrue($result);
+    }
+
+    /**
+     * @depends test_archive_to_storage
+     */
+    public function test_extract_to_pathname_returnvalue_failure(): void {
+        global $CFG;
+
+        $this->resetAfterTest(false);
+
+        $packer = get_file_packer('application/zip');
+
+        $target = make_request_directory();
+
+        $archive = "$CFG->tempdir/noarchive.zip";
+        $result = $packer->extract_to_pathname($archive, $target, null, null, true);
+        $this->assertFalse($result);
+    }
+
+    /**
+     * @depends test_archive_to_storage
+     */
+    public function test_extract_to_storage(): void {
+        global $CFG;
+
+        $this->resetAfterTest(false);
+
+        $packer = get_file_packer('application/zip');
+        $fs = get_file_storage();
+        $context = \context_system::instance();
+
+        $testcontent = file_get_contents($this->testfile);
+
+        $archive = $fs->get_file($context->id, 'phpunit', 'test', 0, '/', 'archive.zip');
+        $this->assertNotEmpty($archive);
+        $result = $packer->extract_to_storage($archive, $context->id, 'phpunit', 'target', 0, '/');
+        $this->assertTrue(is_array($result));
+        $this->assertEquals(count($this->files), count($result));
+        foreach ($this->files as $file => $unused) {
+            $this->assertTrue($result[$file]);
+            $stored_file = $fs->get_file_by_hash(sha1("/$context->id/phpunit/target/0/$file"));
+            $this->assertInstanceOf('stored_file', $stored_file);
+            $this->assertSame($testcontent, $stored_file->get_content());
+        }
+
+        $archive = "$CFG->tempdir/archive.zip";
+        $this->assertFileExists($archive);
+        $result = $packer->extract_to_storage($archive, $context->id, 'phpunit', 'target', 0, '/');
+        $this->assertTrue(is_array($result));
+        $this->assertEquals(count($this->files), count($result));
+        foreach ($this->files as $file => $unused) {
+            $this->assertTrue($result[$file]);
+            $stored_file = $fs->get_file_by_hash(sha1("/$context->id/phpunit/target/0/$file"));
+            $this->assertInstanceOf('stored_file', $stored_file);
+            $this->assertSame($testcontent, $stored_file->get_content());
+        }
+        unlink($archive);
+    }
+
+    /**
+     * @depends test_extract_to_storage
+     */
+    public function test_add_files(): void {
+        global $CFG;
+
+        $this->resetAfterTest(false);
+
+        $packer = get_file_packer('application/zip');
+        $archive = "$CFG->tempdir/archive.zip";
+
+        $this->assertFileDoesNotExist($archive);
+        $packer->archive_to_pathname(array(), $archive);
+        $this->assertFileExists($archive);
+
+        $zip_archive = new zip_archive();
+        $zip_archive->open($archive, file_archive::OPEN);
+        $this->assertEquals(0, $zip_archive->count());
+
+        $zip_archive->add_file_from_string('test.txt', 'test');
+        $zip_archive->close();
+        $zip_archive->open($archive, file_archive::OPEN);
+        $this->assertEquals(1, $zip_archive->count());
+
+        $zip_archive->add_directory('test2');
+        $zip_archive->close();
+        $zip_archive->open($archive, file_archive::OPEN);
+        $files = $zip_archive->list_files();
+        $this->assertCount(2, $files);
+        $this->assertEquals('test.txt', $files[0]->pathname);
+        $this->assertEquals('test2/', $files[1]->pathname);
+
+        $result = $zip_archive->add_file_from_pathname('test.txt', __DIR__.'/nonexistent/file.txt');
+        $this->assertFalse($result);
+        $zip_archive->close();
+        $zip_archive->open($archive, file_archive::OPEN);
+        $this->assertEquals(2, $zip_archive->count());
+        $zip_archive->close();
+
+        unlink($archive);
+    }
+
+    public function test_close_archive(): void {
+        global $CFG;
+
+        $this->resetAfterTest(true);
+
+        $archive = "$CFG->tempdir/archive.zip";
+        $textfile = "$CFG->tempdir/textfile.txt";
+        touch($textfile);
+
+        $this->assertFileDoesNotExist($archive);
+        $this->assertFileExists($textfile);
+
+        // Create archive and close it without files.
+        // (returns true, without any warning).
+        $zip_archive = new zip_archive();
+        $result = $zip_archive->open($archive, file_archive::CREATE);
+        $this->assertTrue($result);
+        $result = $zip_archive->close();
+        $this->assertTrue($result);
+        unlink($archive);
+
+        // Create archive and close it with files.
+        // (returns true, without any warning).
+        $zip_archive = new zip_archive();
+        $result = $zip_archive->open($archive, file_archive::CREATE);
+        $this->assertTrue($result);
+        $result = $zip_archive->add_file_from_string('test.txt', 'test');
+        $this->assertTrue($result);
+        $result = $zip_archive->add_file_from_pathname('test2.txt', $textfile);
+        $result = $zip_archive->close();
+        $this->assertTrue($result);
+        unlink($archive);
+
+        // Create archive and close if forcing error.
+        // (returns true for old PHP versions and
+        // false with warnings for new PHP versions). MDL-51863.
+        $zip_archive = new zip_archive();
+        $result = $zip_archive->open($archive, file_archive::CREATE);
+        $this->assertTrue($result);
+        $result = $zip_archive->add_file_from_string('test.txt', 'test');
+        $this->assertTrue($result);
+        $result = $zip_archive->add_file_from_pathname('test2.txt', $textfile);
+        $this->assertTrue($result);
+        // Delete the file before closing does force close() to fail.
+        unlink($textfile);
+        // Behavior is different between old PHP versions and new ones. Let's detect it.
+        $result = false;
+        try {
+            // Old PHP versions were not printing any warning.
+            $result = $zip_archive->close();
+        } catch (\Exception $e) {
+            // New PHP versions print PHP Warning.
+            $this->assertInstanceOf('PHPUnit\Framework\Error\Warning', $e);
+            $this->assertStringContainsString('ZipArchive::close', $e->getMessage());
+        }
+        // This is crazy, but it shows how some PHP versions do return true.
+        try {
+            // And some PHP versions do return correctly false (5.4.25, 5.6.14...)
+            $this->assertFalse($result);
+        } catch (\Exception $e) {
+            // But others do insist into returning true (5.6.13...). Only can accept them.
+            $this->assertInstanceOf('PHPUnit\Framework\ExpectationFailedException', $e);
+            $this->assertTrue($result);
+        }
+        $this->assertFileDoesNotExist($archive);
+    }
+
+    /**
+     * @depends test_add_files
+     */
+    public function test_open_archive(): void {
+        global $CFG;
+
+        $this->resetAfterTest(true);
+
+        $archive = "$CFG->tempdir/archive.zip";
+
+        $this->assertFileDoesNotExist($archive);
+
+        $zip_archive = new zip_archive();
+        $result = $zip_archive->open($archive, file_archive::OPEN);
+        $this->assertFalse($result);
+        $this->assertDebuggingCalled();
+
+        $zip_archive = new zip_archive();
+        $result = $zip_archive->open($archive, file_archive::CREATE);
+        $this->assertTrue($result);
+        $zip_archive->add_file_from_string('test.txt', 'test');
+        $zip_archive->close();
+        $zip_archive->open($archive, file_archive::OPEN);
+        $this->assertEquals(1, $zip_archive->count());
+
+        $zip_archive = new zip_archive();
+        $result = $zip_archive->open($archive, file_archive::OVERWRITE);
+        $this->assertTrue($result);
+        $zip_archive->add_file_from_string('test2.txt', 'test');
+        $zip_archive->close();
+        $zip_archive->open($archive, file_archive::OPEN);
+        $this->assertEquals(1, $zip_archive->count());
+        $zip_archive->close();
+
+        unlink($archive);
+        $zip_archive = new zip_archive();
+        $result = $zip_archive->open($archive, file_archive::OVERWRITE);
+        $this->assertTrue($result);
+        $zip_archive->add_file_from_string('test2.txt', 'test');
+        $zip_archive->close();
+        $zip_archive->open($archive, file_archive::OPEN);
+        $this->assertEquals(1, $zip_archive->count());
+        $zip_archive->close();
+
+        unlink($archive);
+    }
+
+    /**
+     * Test opening an encrypted archive
+     */
+    public function test_open_encrypted_archive(): void {
+        $this->resetAfterTest();
+
+        // The archive contains a single encrypted "hello.txt" file.
+        $archive = __DIR__ . '/fixtures/passwordis1.zip';
+
+        /** @var \zip_packer $packer */
+        $packer = get_file_packer('application/zip');
+        $result = $packer->extract_to_pathname($archive, make_temp_directory('zip'));
+
+        $this->assertIsArray($result);
+        $this->assertArrayHasKey('hello.txt', $result);
+        $this->assertEquals('Can not read file from zip archive', $result['hello.txt']);
+    }
+
+    /**
+     * Tests the progress reporting.
+     */
+    public function test_file_progress(): void {
+        global $CFG;
+
+        // Set up.
+        $this->resetAfterTest(true);
+        $packer = get_file_packer('application/zip');
+        $archive = "$CFG->tempdir/archive.zip";
+        $context = \context_system::instance();
+
+        // Archive to pathname.
+        $this->progress = array();
+        $result = $packer->archive_to_pathname($this->files, $archive, true, $this);
+        $this->assertTrue($result);
+        // Should send progress at least once per file.
+        $this->assertTrue(count($this->progress) >= count($this->files));
+        // Each progress will be indeterminate.
+        $this->assertEquals(
+                array(file_progress::INDETERMINATE, file_progress::INDETERMINATE),
+                $this->progress[0]);
+
+        // Archive to pathname using entire folder and subfolder instead of file list.
+        unlink($archive);
+        $folder = make_temp_directory('zip_packer_progress');
+        file_put_contents($folder . '/test1.txt', 'hello');
+        $subfolder = $folder . '/sub';
+        check_dir_exists($subfolder);
+        file_put_contents($subfolder . '/test2.txt', 'world');
+        file_put_contents($subfolder . '/test3.txt', 'and');
+        file_put_contents($subfolder . '/test4.txt', 'other');
+        file_put_contents($subfolder . '/test5.txt', 'worlds');
+        $this->progress = array();
+        $result = $packer->archive_to_pathname(array('' => $folder), $archive, true, $this);
+        $this->assertTrue($result);
+        // Should send progress at least once per file.
+        $this->assertTrue(count($this->progress) >= 5);
+
+        // Archive to storage.
+        $this->progress = array();
+        $archivefile = $packer->archive_to_storage($this->files, $context->id,
+                'phpunit', 'test', 0, '/', 'archive.zip', null, true, $this);
+        $this->assertInstanceOf('stored_file', $archivefile);
+        $this->assertTrue(count($this->progress) >= count($this->files));
+        $this->assertEquals(
+                array(file_progress::INDETERMINATE, file_progress::INDETERMINATE),
+                $this->progress[0]);
+
+        // Extract to pathname.
+        $this->progress = array();
+        $target = "$CFG->tempdir/test/";
+        check_dir_exists($target);
+        $result = $packer->extract_to_pathname($archive, $target, null, $this);
+        remove_dir($target);
+        $this->assertEquals(count($this->files), count($result));
+        $this->assertTrue(count($this->progress) >= count($this->files));
+        $this->check_progress_toward_max();
+
+        // Extract to storage (from storage).
+        $this->progress = array();
+        $result = $packer->extract_to_storage($archivefile, $context->id,
+                'phpunit', 'target', 0, '/', null, $this);
+        $this->assertEquals(count($this->files), count($result));
+        $this->assertTrue(count($this->progress) >= count($this->files));
+        $this->check_progress_toward_max();
+
+        // Extract to storage (from path).
+        $this->progress = array();
+        $result = $packer->extract_to_storage($archive, $context->id,
+                'phpunit', 'target', 0, '/', null, $this);
+        $this->assertEquals(count($this->files), count($result));
+        $this->assertTrue(count($this->progress) >= count($this->files));
+        $this->check_progress_toward_max();
+
+        // Wipe created disk file.
+        unlink($archive);
+    }
+
+    /**
+     * Checks that progress reported is numeric rather than indeterminate,
+     * and follows the progress reporting rules.
+     */
+    private function check_progress_toward_max() {
+        $lastvalue = -1;
+        foreach ($this->progress as $progressitem) {
+            list($value, $max) = $progressitem;
+            $this->assertNotEquals(file_progress::INDETERMINATE, $max);
+            $this->assertTrue($value <= $max);
+            $this->assertTrue($value >= $lastvalue);
+            $lastvalue = $value;
+        }
+    }
+
+    /**
+     * Handles file_progress interface.
+     *
+     * @param int $progress
+     * @param int $max
+     */
+    public function progress($progress = file_progress::INDETERMINATE, $max = file_progress::INDETERMINATE) {
+        $this->progress[] = array($progress, $max);
+    }
+}

lib/filestorage/tgz_extractor.php

@@ -0,0 +1,565 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Implementation of .tar.gz extractor. Handles extraction of .tar.gz files.
+ * Do not call directly; use methods in tgz_packer.
+ *
+ * @see tgz_packer
+ * @package core_files
+ * @copyright 2013 The Open University
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Extracts .tar.gz files (POSIX format).
+ */
+class tgz_extractor {
+    /**
+     * @var int When writing data, the system writes blocks of this size.
+     */
+    const WRITE_BLOCK_SIZE = 65536;
+    /**
+     * @var int When reading data, the system reads blocks of this size.
+     */
+    const READ_BLOCK_SIZE = 65536;
+    /**
+     * @var stored_file File object for archive.
+     */
+    protected $storedfile;
+    /**
+     * @var string OS path for archive.
+     */
+    protected $ospath;
+    /**
+     * @var int Number of files (-1 if not known).
+     */
+    protected $numfiles;
+    /**
+     * @var int Number of files processed so far.
+     */
+    protected $donefiles;
+    /**
+     * @var string Current file path within archive.
+     */
+    protected $currentarchivepath;
+    /**
+     * @var string Full path to current file.
+     */
+    protected $currentfile;
+    /**
+     * @var int Size of current file in bytes.
+     */
+    protected $currentfilesize;
+    /**
+     * @var int Number of bytes of current file already written into buffer.
+     */
+    protected $currentfileprocessed;
+    /**
+     * @var resource File handle to current file.
+     */
+    protected $currentfp;
+    /**
+     * @var int Modified time of current file.
+     */
+    protected $currentmtime;
+    /**
+     * @var string Buffer containing file data awaiting write.
+     */
+    protected $filebuffer;
+    /**
+     * @var int Current length of buffer in bytes.
+     */
+    protected $filebufferlength;
+    /**
+     * @var array Results array of all files processed.
+     */
+    protected $results;
+
+    /**
+     * @var array In list mode, content of the list; outside list mode, null.
+     */
+    protected $listresults = null;
+
+    /**
+     * @var int Whether listing or extracting.
+     */
+    protected $mode = self::MODE_EXTRACT;
+
+    /**
+     * @var int If extracting (default).
+     */
+    const MODE_EXTRACT = 0;
+
+    /**
+     * @var int Listing contents.
+     */
+    const MODE_LIST = 1;
+
+    /**
+     * @var int Listing contents; list now complete.
+     */
+    const MODE_LIST_COMPLETE = 2;
+
+    /**
+     * Constructor.
+     *
+     * @param stored_file|string $archivefile Moodle file or OS path to archive
+     */
+    public function __construct($archivefile) {
+        if (is_a($archivefile, 'stored_file')) {
+            $this->storedfile = $archivefile;
+        } else {
+            $this->ospath = $archivefile;
+        }
+    }
+
+    /**
+     * Extracts the archive.
+     *
+     * @param tgz_extractor_handler $handler Will be called for extracted files
+     * @param file_progress $progress Optional progress reporting
+     * @return array Array from archive path => true of processed files
+     * @throws moodle_exception If there is any error processing the archive
+     */
+    public function extract(tgz_extractor_handler $handler, file_progress $progress = null) {
+        $this->mode = self::MODE_EXTRACT;
+        $this->extract_or_list($handler, $progress);
+        $results = $this->results;
+        unset($this->results);
+        return $results;
+    }
+
+    /**
+     * Extracts or lists the archive depending on $this->listmode.
+     *
+     * @param tgz_extractor_handler $handler Optional handler
+     * @param file_progress $progress Optional progress reporting
+     * @throws moodle_exception If there is any error processing the archive
+     */
+    protected function extract_or_list(tgz_extractor_handler $handler = null, file_progress $progress = null) {
+        // Open archive.
+        if ($this->storedfile) {
+            $gz = $this->storedfile->get_content_file_handle(stored_file::FILE_HANDLE_GZOPEN);
+            // Estimate number of read-buffers (64KB) in file. Guess that the
+            // uncompressed size is 2x compressed size. Add one just to ensure
+            // it's non-zero.
+            $estimatedbuffers = ($this->storedfile->get_filesize() * 2 / self::READ_BLOCK_SIZE) + 1;
+        } else {
+            $gz = gzopen($this->ospath, 'rb');
+            $estimatedbuffers = (filesize($this->ospath) * 2 / self::READ_BLOCK_SIZE) + 1;
+        }
+        if (!$gz) {
+            throw new moodle_exception('errorprocessingarchive', '', '', null,
+                    'Failed to open gzip file');
+        }
+
+        // Calculate how much progress to report per buffer read.
+        $progressperbuffer = (int)(tgz_packer::PROGRESS_MAX / $estimatedbuffers);
+
+        // Process archive in 512-byte blocks (but reading 64KB at a time).
+        $buffer = '';
+        $bufferpos = 0;
+        $bufferlength = 0;
+        $this->numfiles = -1;
+        $read = 0;
+        $done = 0;
+        $beforeprogress = -1;
+        while (true) {
+            if ($bufferpos == $bufferlength) {
+                $buffer = gzread($gz, self::READ_BLOCK_SIZE);
+                $bufferpos = 0;
+                $bufferlength = strlen($buffer);
+                if ($bufferlength == 0) {
+                    // EOF.
+                    break;
+                }
+
+                // Report progress if enabled.
+                if ($progress) {
+                    if ($this->numfiles === -1) {
+                        // If we don't know the number of files, do an estimate based
+                        // on number of buffers read.
+                        $done += $progressperbuffer;
+                        if ($done >= tgz_packer::PROGRESS_MAX) {
+                            $done = tgz_packer::PROGRESS_MAX - 1;
+                        }
+                        $progress->progress($done, tgz_packer::PROGRESS_MAX);
+                    } else {
+                        // Once we know the number of files, use this.
+                        if ($beforeprogress === -1) {
+                            $beforeprogress = $done;
+                        }
+                        // Calculate progress as whatever progress we reported
+                        // before we knew how many files there were (might be 0)
+                        // plus a proportion of the number of files out of the
+                        // remaining progress value.
+                        $done = $beforeprogress + (int)(($this->donefiles / $this->numfiles) *
+                                (tgz_packer::PROGRESS_MAX - $beforeprogress));
+                    }
+                    $progress->progress($done, tgz_packer::PROGRESS_MAX);
+                }
+            }
+
+            $block = substr($buffer, $bufferpos, tgz_packer::TAR_BLOCK_SIZE);
+            if ($this->currentfile) {
+                $this->process_file_block($block, $handler);
+            } else {
+                $this->process_header($block, $handler);
+            }
+
+            // When listing, if we read an index file, we abort archive processing.
+            if ($this->mode === self::MODE_LIST_COMPLETE) {
+                break;
+            }
+
+            $bufferpos += tgz_packer::TAR_BLOCK_SIZE;
+            $read++;
+        }
+
+        // Close archive and finish.
+        gzclose($gz);
+    }
+
+    /**
+     * Lists files in the archive, either using the index file (if present),
+     * or by basically extracting the whole thing if there isn't an index file.
+     *
+     * @return array Array of file listing results:
+     */
+    public function list_files() {
+        $this->listresults = array();
+        $this->mode = self::MODE_LIST;
+        $this->extract_or_list();
+        $listresults = $this->listresults;
+        $this->listresults = null;
+        return $listresults;
+    }
+
+    /**
+     * Process 512-byte header block.
+     *
+     * @param string $block Tar block
+     * @param tgz_extractor_handler $handler Will be called for extracted files
+     */
+    protected function process_header($block, $handler) {
+        // If the block consists entirely of nulls, ignore it. (This happens
+        // twice at end of archive.)
+        if ($block === str_pad('', tgz_packer::TAR_BLOCK_SIZE, "\0")) {
+            return;
+        }
+
+        // struct header_posix_ustar {
+        //    char name[100];
+        $name = rtrim(substr($block, 0, 100), "\0");
+
+        //    char mode[8];
+        //    char uid[8];
+        //    char gid[8];
+        //    char size[12];
+        $filesize = octdec(substr($block, 124, 11));
+
+        //    char mtime[12];
+        $mtime = octdec(substr($block, 136, 11));
+
+        //    char checksum[8];
+        //    char typeflag[1];
+        $typeflag = substr($block, 156, 1);
+
+        //    char linkname[100];
+        //    char magic[6];
+        $magic = substr($block, 257, 6);
+        if ($magic !== "ustar\0" && $magic !== "ustar ") {
+            // There are two checks above; the first is the correct POSIX format
+            // and the second is for GNU tar default format.
+            throw new moodle_exception('errorprocessingarchive', '', '', null,
+                    'Header does not have POSIX ustar magic string');
+        }
+
+        //    char version[2];
+        //    char uname[32];
+        //    char gname[32];
+        //    char devmajor[8];
+        //    char devminor[8];
+        //    char prefix[155];
+        $prefix = rtrim(substr($block, 345, 155), "\0");
+
+        //    char pad[12];
+        // };
+
+        $archivepath = ltrim($prefix . '/' . $name, '/');
+
+        // For security, ensure there is no .. folder in the archivepath.
+        $archivepath = clean_param($archivepath, PARAM_PATH);
+
+        // Handle file depending on the type.
+        switch ($typeflag) {
+            case '1' :
+            case '2' :
+            case '3' :
+            case '4' :
+            case '6' :
+            case '7' :
+                // Ignore these special cases.
+                break;
+
+            case '5' :
+                // Directory.
+                if ($this->mode === self::MODE_LIST) {
+                    $this->listresults[] = (object)array(
+                            'original_pathname' => $archivepath,
+                            'pathname' => $archivepath,
+                            'mtime' => $mtime,
+                            'is_directory' => true,
+                            'size' => 0);
+                } else if ($handler->tgz_directory($archivepath, $mtime)) {
+                    $this->results[$archivepath] = true;
+                }
+                break;
+
+            default:
+                // All other values treated as normal file.
+                $this->start_current_file($archivepath, $filesize, $mtime, $handler);
+                break;
+        }
+    }
+
+    /**
+     * Processes one 512-byte block of an existing file.
+     *
+     * @param string $block Data block
+     * @param tgz_extractor_handler $handler Will be called for extracted files
+     */
+    protected function process_file_block($block, tgz_extractor_handler $handler = null) {
+        // Write block into buffer.
+        $blocksize = tgz_packer::TAR_BLOCK_SIZE;
+        if ($this->currentfileprocessed + tgz_packer::TAR_BLOCK_SIZE > $this->currentfilesize) {
+            // Partial block at end of file.
+            $blocksize = $this->currentfilesize - $this->currentfileprocessed;
+            $this->filebuffer .= substr($block, 0, $blocksize);
+        } else {
+            // Full-length block.
+            $this->filebuffer .= $block;
+        }
+        $this->filebufferlength += $blocksize;
+        $this->currentfileprocessed += $blocksize;
+
+        // Write block to file if necessary.
+        $eof = $this->currentfileprocessed == $this->currentfilesize;
+        if ($this->filebufferlength >= self::WRITE_BLOCK_SIZE || $eof) {
+            // Except when skipping the file, write it out.
+            if ($this->currentfile !== true) {
+                if (!fwrite($this->currentfp, $this->filebuffer)) {
+                    throw new moodle_exception('errorprocessingarchive', '', '', null,
+                            'Failed to write buffer to output file: ' . $this->currentfile);
+                }
+            }
+            $this->filebuffer = '';
+            $this->filebufferlength = 0;
+        }
+
+        // If file is finished, close it.
+        if ($eof) {
+            $this->close_current_file($handler);
+        }
+    }
+
+    /**
+     * Starts processing a file from archive.
+     *
+     * @param string $archivepath Path inside archive
+     * @param int $filesize Size in bytes
+     * @param int $mtime File-modified time
+     * @param tgz_extractor_handler $handler Will be called for extracted files
+     * @throws moodle_exception
+     */
+    protected function start_current_file($archivepath, $filesize, $mtime,
+            tgz_extractor_handler $handler = null) {
+        global $CFG;
+
+        $this->currentarchivepath = $archivepath;
+        $this->currentmtime = $mtime;
+        $this->currentfilesize = $filesize;
+        $this->currentfileprocessed = 0;
+
+        if ($archivepath === tgz_packer::ARCHIVE_INDEX_FILE) {
+            // For index file, store in temp directory.
+            $tempfolder = $CFG->tempdir . '/core_files';
+            check_dir_exists($tempfolder);
+            $this->currentfile = tempnam($tempfolder, '.index');
+        } else {
+            if ($this->mode === self::MODE_LIST) {
+                // If listing, add to list.
+                $this->listresults[] = (object)array(
+                        'original_pathname' => $archivepath,
+                        'pathname' => $archivepath,
+                        'mtime' => $mtime,
+                        'is_directory' => false,
+                        'size' => $filesize);
+
+                // Discard file.
+                $this->currentfile = true;
+            } else {
+                // For other files, ask handler for location.
+                $this->currentfile = $handler->tgz_start_file($archivepath);
+                if ($this->currentfile === null) {
+                    // This indicates that we are discarding the current file.
+                    $this->currentfile = true;
+                }
+            }
+        }
+        $this->filebuffer = '';
+        $this->filebufferlength = 0;
+
+        // Open file.
+        if ($this->currentfile !== true) {
+            $this->currentfp = fopen($this->currentfile, 'wb');
+            if (!$this->currentfp) {
+                throw new moodle_exception('errorprocessingarchive', '', '', null,
+                        'Failed to open output file: ' . $this->currentfile);
+            }
+        } else {
+            $this->currentfp = null;
+        }
+
+        // If it has no size, close it right away.
+        if ($filesize == 0) {
+            $this->close_current_file($handler);
+        }
+    }
+
+    /**
+     * Closes the current file, calls handler, and sets up data.
+     *
+     * @param tgz_extractor_handler $handler Will be called for extracted files
+     * @throws moodle_exception If there is an error closing it
+     */
+    protected function close_current_file($handler) {
+        if ($this->currentfp !== null) {
+            if (!fclose($this->currentfp)) {
+                throw new moodle_exception('errorprocessingarchive', '', '', null,
+                        'Failed to close output file: ' .  $this->currentfile);
+            }
+
+            // At this point we should touch the file to set its modified
+            // time to $this->currentmtime. However, when extracting to the
+            // temp directory, cron will delete files more than a week old,
+            // so to avoid problems we leave all files at their current time.
+        }
+
+        if ($this->currentarchivepath === tgz_packer::ARCHIVE_INDEX_FILE) {
+            if ($this->mode === self::MODE_LIST) {
+                // When listing array, use the archive index to produce the list.
+                $index = file($this->currentfile);
+                $ok = true;
+                foreach ($index as $num => $value) {
+                    // For first line (header), check it's valid then skip it.
+                    if ($num == 0) {
+                        if (preg_match('~^' . preg_quote(tgz_packer::ARCHIVE_INDEX_COUNT_PREFIX) . '~', $value)) {
+                            continue;
+                        } else {
+                            // Not valid, better ignore the file.
+                            $ok = false;
+                            break;
+                        }
+                    }
+                    // Split on tabs and store in results array.
+                    $values = explode("\t", trim($value));
+                    $this->listresults[] = (object)array(
+                        'original_pathname' => $values[0],
+                        'pathname' => $values[0],
+                        'mtime' => ($values[3] === '?' ? tgz_packer::DEFAULT_TIMESTAMP : (int)$values[3]),
+                        'is_directory' => $values[1] === 'd',
+                        'size' => (int)$values[2]);
+                }
+                if ($ok) {
+                    $this->mode = self::MODE_LIST_COMPLETE;
+                }
+                unlink($this->currentfile);
+            } else {
+                // For index file, get number of files and delete temp file.
+                $contents = file_get_contents($this->currentfile, false, null, 0, 128);
+                $matches = array();
+                if (preg_match('~^' . preg_quote(tgz_packer::ARCHIVE_INDEX_COUNT_PREFIX) .
+                        '([0-9]+)~', $contents, $matches)) {
+                    $this->numfiles = (int)$matches[1];
+                }
+                unlink($this->currentfile);
+            }
+        } else {
+            // Report to handler and put in results.
+            if ($this->currentfp !== null) {
+                $handler->tgz_end_file($this->currentarchivepath, $this->currentfile);
+                $this->results[$this->currentarchivepath] = true;
+            }
+            $this->donefiles++;
+        }
+
+        // No longer have a current file.
+        $this->currentfp = null;
+        $this->currentfile = null;
+        $this->currentarchivepath = null;
+    }
+
+}
+
+/**
+ * Interface for callback from tgz_extractor::extract.
+ *
+ * The file functions will be called (in pairs tgz_start_file, tgz_end_file) for
+ * each file in the archive. (There is only one exception, the special
+ * .ARCHIVE_INDEX file which is not reported to the handler.)
+ *
+ * The directory function is called whenever the archive contains a directory
+ * entry.
+ */
+interface tgz_extractor_handler {
+    /**
+     * Called when the system begins to extract a file. At this point, the
+     * handler must decide where on disk the extracted file should be located.
+     * This can be a temporary location or final target, as preferred.
+     *
+     * The handler can request for files to be skipped, in which case no data
+     * will be written and tgz_end_file will not be called.
+     *
+     * @param string $archivepath Path and name of file within archive
+     * @return string Location for output file in filesystem, or null to skip file
+     */
+    public function tgz_start_file($archivepath);
+
+    /**
+     * Called when the system has finished extracting a file. The handler can
+     * now process the extracted file if required.
+     *
+     * @param string $archivepath Path and name of file within archive
+     * @param string $realpath Path in filesystem (from tgz_start_file return)
+     * @return bool True to continue processing, false to abort archive extract
+     */
+    public function tgz_end_file($archivepath, $realpath);
+
+    /**
+     * Called when a directory entry is found in the archive.
+     *
+     * The handler can create a corresponding directory if required.
+     *
+     * @param string $archivepath Path and name of directory within archive
+     * @param int $mtime Modified time of directory
+     * @return bool True if directory was created, false if skipped
+     */
+    public function tgz_directory($archivepath, $mtime);
+}

lib/filestorage/tgz_packer.php

@@ -0,0 +1,902 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Implementation of .tar.gz packer.
+ *
+ * A limited subset of the .tar format is supported. This packer can open files
+ * that it wrote, but may not be able to open files from other sources,
+ * especially if they use extensions. There are restrictions on file
+ * length and character set of filenames.
+ *
+ * We generate POSIX-compliant ustar files. As a result, the following
+ * restrictions apply to archive paths:
+ *
+ * - Filename may not be more than 100 characters.
+ * - Total of path + filename may not be more than 256 characters.
+ * - For path more than 155 characters it may or may not work.
+ * - May not contain non-ASCII characters.
+ *
+ * @package core_files
+ * @copyright 2013 The Open University
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+require_once("$CFG->libdir/filestorage/file_packer.php");
+require_once("$CFG->libdir/filestorage/tgz_extractor.php");
+
+/**
+ * Utility class - handles all packing/unpacking of .tar.gz files.
+ *
+ * @package core_files
+ * @category files
+ * @copyright 2013 The Open University
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class tgz_packer extends file_packer {
+    /**
+     * @var int Default timestamp used where unknown (Jan 1st 2013 00:00)
+     */
+    const DEFAULT_TIMESTAMP = 1356998400;
+
+    /**
+     * @var string Name of special archive index file added by Moodle.
+     */
+    const ARCHIVE_INDEX_FILE = '.ARCHIVE_INDEX';
+
+    /**
+     * @var string Required text at start of archive index file before file count.
+     */
+    const ARCHIVE_INDEX_COUNT_PREFIX = 'Moodle archive file index. Count: ';
+
+    /**
+     * @var bool If true, includes .ARCHIVE_INDEX file in root of tar file.
+     */
+    protected $includeindex = true;
+
+    /**
+     * @var int Max value for total progress.
+     */
+    const PROGRESS_MAX = 1000000;
+
+    /**
+     * @var int Tar files have a fixed block size of 512 bytes.
+     */
+    const TAR_BLOCK_SIZE = 512;
+
+    /**
+     * Archive files and store the result in file storage.
+     *
+     * Any existing file at that location will be overwritten.
+     *
+     * @param array $files array from archive path => pathname or stored_file
+     * @param int $contextid context ID
+     * @param string $component component
+     * @param string $filearea file area
+     * @param int $itemid item ID
+     * @param string $filepath file path
+     * @param string $filename file name
+     * @param int $userid user ID
+     * @param bool $ignoreinvalidfiles true means ignore missing or invalid files, false means abort on any error
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return stored_file|bool false if error stored_file instance if ok
+     * @throws file_exception If file operations fail
+     * @throws coding_exception If any archive paths do not meet the restrictions
+     */
+    public function archive_to_storage(array $files, $contextid,
+            $component, $filearea, $itemid, $filepath, $filename,
+            $userid = null, $ignoreinvalidfiles = true, file_progress $progress = null) {
+        global $CFG;
+
+        // Set up a temporary location for the file.
+        $tempfolder = $CFG->tempdir . '/core_files';
+        check_dir_exists($tempfolder);
+        $tempfile = tempnam($tempfolder, '.tgz');
+
+        // Archive to the given path.
+        if ($result = $this->archive_to_pathname($files, $tempfile, $ignoreinvalidfiles, $progress)) {
+            // If there is an existing file, delete it.
+            $fs = get_file_storage();
+            if ($existing = $fs->get_file($contextid, $component, $filearea, $itemid, $filepath, $filename)) {
+                $existing->delete();
+            }
+            $filerecord = array('contextid' => $contextid, 'component' => $component,
+                    'filearea' => $filearea, 'itemid' => $itemid, 'filepath' => $filepath,
+                    'filename' => $filename, 'userid' => $userid, 'mimetype' => 'application/x-tgz');
+            self::delete_existing_file_record($fs, $filerecord);
+            $result = $fs->create_file_from_pathname($filerecord, $tempfile);
+        }
+
+        // Delete the temporary file (if created) and return.
+        @unlink($tempfile);
+        return $result;
+    }
+
+    /**
+     * Wrapper function useful for deleting an existing file (if present) just
+     * before creating a new one.
+     *
+     * @param file_storage $fs File storage
+     * @param array $filerecord File record in same format used to create file
+     */
+    public static function delete_existing_file_record(file_storage $fs, array $filerecord) {
+        if ($existing = $fs->get_file($filerecord['contextid'], $filerecord['component'],
+                $filerecord['filearea'], $filerecord['itemid'], $filerecord['filepath'],
+                $filerecord['filename'])) {
+            $existing->delete();
+        }
+    }
+
+    /**
+     * By default, the .tar file includes a .ARCHIVE_INDEX file as its first
+     * entry. This makes list_files much faster and allows for better progress
+     * reporting.
+     *
+     * If you need to disable the inclusion of this file, use this function
+     * before calling one of the archive_xx functions.
+     *
+     * @param bool $includeindex If true, includes index
+     */
+    public function set_include_index($includeindex) {
+        $this->includeindex = $includeindex;
+    }
+
+    /**
+     * Archive files and store the result in an OS file.
+     *
+     * @param array $files array from archive path => pathname or stored_file
+     * @param string $archivefile path to target zip file
+     * @param bool $ignoreinvalidfiles true means ignore missing or invalid files, false means abort on any error
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return bool true if file created, false if not
+     * @throws coding_exception If any archive paths do not meet the restrictions
+     */
+    public function archive_to_pathname(array $files, $archivefile,
+            $ignoreinvalidfiles=true, file_progress $progress = null) {
+        // Open .gz file.
+        if (!($gz = gzopen($archivefile, 'wb'))) {
+            return false;
+        }
+        try {
+            // Because we update how we calculate progress after we already
+            // analyse the directory list, we can't just use a number of files
+            // as progress. Instead, progress always goes to PROGRESS_MAX
+            // and we do estimates as a proportion of that. To begin with,
+            // assume that counting files will be 10% of the work, so allocate
+            // one-tenth of PROGRESS_MAX to the total of all files.
+            if ($files) {
+                $progressperfile = (int)(self::PROGRESS_MAX / (count($files) * 10));
+            } else {
+                // If there are no files, avoid divide by zero.
+                $progressperfile = 1;
+            }
+            $done = 0;
+
+            // Expand the provided files into a complete list of single files.
+            $expandedfiles = array();
+            foreach ($files as $archivepath => $file) {
+                // Update progress if required.
+                if ($progress) {
+                    $progress->progress($done, self::PROGRESS_MAX);
+                }
+                $done += $progressperfile;
+
+                if (is_null($file)) {
+                    // Empty directory record. Ensure it ends in a /.
+                    if (!preg_match('~/$~', $archivepath)) {
+                        $archivepath .= '/';
+                    }
+                    $expandedfiles[$archivepath] = null;
+                } else if (is_string($file)) {
+                    // File specified as path on disk.
+                    if (!$this->list_files_path($expandedfiles, $archivepath, $file,
+                            $progress, $done)) {
+                        gzclose($gz);
+                        unlink($archivefile);
+                        return false;
+                    }
+                } else if (is_array($file)) {
+                    // File specified as raw content in array.
+                    $expandedfiles[$archivepath] = $file;
+                } else {
+                    // File specified as stored_file object.
+                    $this->list_files_stored($expandedfiles, $archivepath, $file);
+                }
+            }
+
+            // Store the list of files as a special file that is first in the
+            // archive. This contains enough information to implement list_files
+            // if required later.
+            $list = self::ARCHIVE_INDEX_COUNT_PREFIX . count($expandedfiles) . "\n";
+            $sizes = array();
+            $mtimes = array();
+            foreach ($expandedfiles as $archivepath => $file) {
+                // Check archivepath doesn't contain any non-ASCII characters.
+                if (!preg_match('~^[\x00-\xff]*$~', $archivepath)) {
+                    throw new coding_exception(
+                            'Non-ASCII paths not supported: ' . $archivepath);
+                }
+
+                // Build up the details.
+                $type = 'f';
+                $mtime = '?';
+                if (is_null($file)) {
+                    $type = 'd';
+                    $size = 0;
+                } else if (is_string($file)) {
+                    $stat = stat($file);
+                    $mtime = (int)$stat['mtime'];
+                    $size = (int)$stat['size'];
+                } else if (is_array($file)) {
+                    $size = (int)strlen(reset($file));
+                } else {
+                    $mtime = (int)$file->get_timemodified();
+                    $size = (int)$file->get_filesize();
+                }
+                $sizes[$archivepath] = $size;
+                $mtimes[$archivepath] = $mtime;
+
+                // Write a line in the index.
+                $list .= "$archivepath\t$type\t$size\t$mtime\n";
+            }
+
+            // The index file is optional; only write into archive if needed.
+            if ($this->includeindex) {
+                // Put the index file into the archive.
+                $this->write_tar_entry($gz, self::ARCHIVE_INDEX_FILE, null, strlen($list), '?', $list);
+            }
+
+            // Update progress ready for main stage.
+            $done = (int)(self::PROGRESS_MAX / 10);
+            if ($progress) {
+                $progress->progress($done, self::PROGRESS_MAX);
+            }
+            if ($expandedfiles) {
+                // The remaining 9/10ths of progress represents these files.
+                $progressperfile = (int)((9 * self::PROGRESS_MAX) / (10 * count($expandedfiles)));
+            } else {
+                $progressperfile = 1;
+            }
+
+            // Actually write entries for each file/directory.
+            foreach ($expandedfiles as $archivepath => $file) {
+                if (is_null($file)) {
+                    // Null entry indicates a directory.
+                    $this->write_tar_entry($gz, $archivepath, null,
+                            $sizes[$archivepath], $mtimes[$archivepath]);
+                } else if (is_string($file)) {
+                    // String indicates an OS file.
+                    $this->write_tar_entry($gz, $archivepath, $file,
+                            $sizes[$archivepath], $mtimes[$archivepath], null, $progress, $done);
+                } else if (is_array($file)) {
+                    // Array indicates in-memory data.
+                    $data = reset($file);
+                    $this->write_tar_entry($gz, $archivepath, null,
+                            $sizes[$archivepath], $mtimes[$archivepath], $data, $progress, $done);
+                } else {
+                    // Stored_file object.
+                    $this->write_tar_entry($gz, $archivepath, $file->get_content_file_handle(),
+                            $sizes[$archivepath], $mtimes[$archivepath], null, $progress, $done);
+                }
+                $done += $progressperfile;
+                if ($progress) {
+                    $progress->progress($done, self::PROGRESS_MAX);
+                }
+            }
+
+            // Finish tar file with two empty 512-byte records.
+            gzwrite($gz, str_pad('', 2 * self::TAR_BLOCK_SIZE, "\x00"));
+            gzclose($gz);
+            return true;
+        } catch (Exception $e) {
+            // If there is an exception, delete the in-progress file.
+            gzclose($gz);
+            unlink($archivefile);
+            throw $e;
+        }
+    }
+
+    /**
+     * Writes a single tar file to the archive, including its header record and
+     * then the file contents.
+     *
+     * @param resource $gz Gzip file
+     * @param string $archivepath Full path of file within archive
+     * @param string|resource $file Full path of file on disk or file handle or null if none
+     * @param int $size Size or 0 for directories
+     * @param int|string $mtime Time or ? if unknown
+     * @param string $content Actual content of file to write (null if using $filepath)
+     * @param file_progress $progress Progress indicator or null if none
+     * @param int $done Value for progress indicator
+     * @return bool True if OK
+     * @throws coding_exception If names aren't valid
+     */
+    protected function write_tar_entry($gz, $archivepath, $file, $size, $mtime, $content = null,
+            file_progress $progress = null, $done = 0) {
+        // Header based on documentation of POSIX ustar format from:
+        // http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5&manpath=FreeBSD+8-current .
+
+        // For directories, ensure name ends in a slash.
+        $directory = false;
+        if ($size === 0 && is_null($file)) {
+            $directory = true;
+            if (!preg_match('~/$~', $archivepath)) {
+                $archivepath .= '/';
+            }
+            $mode = '755';
+        } else {
+            $mode = '644';
+        }
+
+        // Split archivepath into name and prefix.
+        $name = $archivepath;
+        $prefix = '';
+        while (strlen($name) > 100) {
+            $slash = strpos($name, '/');
+            if ($slash === false) {
+                throw new coding_exception(
+                        'Name cannot fit length restrictions (> 100 characters): ' . $archivepath);
+            }
+
+            if ($prefix !== '') {
+                $prefix .= '/';
+            }
+            $prefix .= substr($name, 0, $slash);
+            $name = substr($name, $slash + 1);
+            if (strlen($prefix) > 155) {
+                throw new coding_exception(
+                        'Name cannot fit length restrictions (path too long): ' . $archivepath);
+            }
+        }
+
+        // Checksum performance is a bit slow because of having to call 'ord'
+        // lots of times (it takes about 1/3 the time of the actual gzwrite
+        // call). To improve performance of checksum calculation, we will
+        // store all the non-zero, non-fixed bytes that need adding to the
+        // checksum, and checksum only those bytes.
+        $forchecksum = $name;
+
+        // struct header_posix_ustar {
+        //    char name[100];
+        $header = str_pad($name, 100, "\x00");
+
+        //    char mode[8];
+        //    char uid[8];
+        //    char gid[8];
+        $header .= '0000' . $mode . "\x000000000\x000000000\x00";
+        $forchecksum .= $mode;
+
+        //    char size[12];
+        $octalsize = decoct($size);
+        if (strlen($octalsize) > 11) {
+            throw new coding_exception(
+                    'File too large for .tar file: ' . $archivepath . ' (' . $size . ' bytes)');
+        }
+        $paddedsize = str_pad($octalsize, 11, '0', STR_PAD_LEFT);
+        $forchecksum .= $paddedsize;
+        $header .= $paddedsize . "\x00";
+
+        //    char mtime[12];
+        if ($mtime === '?') {
+            // Use a default timestamp rather than zero; GNU tar outputs
+            // warnings about zeroes here.
+            $mtime = self::DEFAULT_TIMESTAMP;
+        }
+        $octaltime = decoct($mtime);
+        $paddedtime = str_pad($octaltime, 11, '0', STR_PAD_LEFT);
+        $forchecksum .= $paddedtime;
+        $header .= $paddedtime . "\x00";
+
+        //    char checksum[8];
+        // Checksum needs to be completed later.
+        $header .= '        ';
+
+        //    char typeflag[1];
+        $typeflag = $directory ? '5' : '0';
+        $forchecksum .= $typeflag;
+        $header .= $typeflag;
+
+        //    char linkname[100];
+        $header .= str_pad('', 100, "\x00");
+
+        //    char magic[6];
+        //    char version[2];
+        $header .= "ustar\x0000";
+
+        //    char uname[32];
+        //    char gname[32];
+        //    char devmajor[8];
+        //    char devminor[8];
+        $header .= str_pad('', 80, "\x00");
+
+        //    char prefix[155];
+        //    char pad[12];
+        $header .= str_pad($prefix, 167, "\x00");
+        $forchecksum .= $prefix;
+
+        // };
+
+        // We have now calculated the header, but without the checksum. To work
+        // out the checksum, sum all the bytes that aren't fixed or zero, and add
+        // to a standard value that contains all the fixed bytes.
+
+        // The fixed non-zero bytes are:
+        //
+        // '000000000000000000        ustar00'
+        // mode (except 3 digits), uid, gid, checksum space, magic number, version
+        //
+        // To calculate the number, call the calculate_checksum function on the
+        // above string. The result is 1775.
+        $checksum = 1775 + self::calculate_checksum($forchecksum);
+
+        $octalchecksum = str_pad(decoct($checksum), 6, '0', STR_PAD_LEFT) . "\x00 ";
+
+        // Slot it into place in the header.
+        $header = substr($header, 0, 148) . $octalchecksum . substr($header, 156);
+
+        if (strlen($header) != self::TAR_BLOCK_SIZE) {
+            throw new coding_exception('Header block wrong size!!!!!');
+        }
+
+        // Awesome, now write out the header.
+        gzwrite($gz, $header);
+
+        // Special pre-handler for OS filename.
+        if (is_string($file)) {
+            $file = fopen($file, 'rb');
+            if (!$file) {
+                return false;
+            }
+        }
+
+        if ($content !== null) {
+            // Write in-memory content if any.
+            if (strlen($content) !== $size) {
+                throw new coding_exception('Mismatch between provided sizes: ' . $archivepath);
+            }
+            gzwrite($gz, $content);
+        } else if ($file !== null) {
+            // Write file content if any, using a 64KB buffer.
+            $written = 0;
+            $chunks = 0;
+            while (true) {
+                $data = fread($file, 65536);
+                if ($data === false || strlen($data) == 0) {
+                    break;
+                }
+                $written += gzwrite($gz, $data);
+
+                // After every megabyte of large files, update the progress
+                // tracker (so there are no long gaps without progress).
+                $chunks++;
+                if ($chunks == 16) {
+                    $chunks = 0;
+                    if ($progress) {
+                        // This call always has the same values, but that gives
+                        // the tracker a chance to indicate indeterminate
+                        // progress and output something to avoid timeouts.
+                        $progress->progress($done, self::PROGRESS_MAX);
+                    }
+                }
+            }
+            fclose($file);
+
+            if ($written !== $size) {
+                throw new coding_exception('Mismatch between provided sizes: ' . $archivepath .
+                        ' (was ' . $written . ', expected ' . $size . ')');
+            }
+        } else if ($size != 0) {
+            throw new coding_exception('Missing data file handle for non-empty file');
+        }
+
+        // Pad out final 512-byte block in file, if applicable.
+        $leftover = self::TAR_BLOCK_SIZE - ($size % self::TAR_BLOCK_SIZE);
+        if ($leftover == 512) {
+            $leftover = 0;
+        } else {
+            gzwrite($gz, str_pad('', $leftover, "\x00"));
+        }
+
+        return true;
+    }
+
+    /**
+     * Calculates a checksum by summing all characters of the binary string
+     * (treating them as unsigned numbers).
+     *
+     * @param string $str Input string
+     * @return int Checksum
+     */
+    protected static function calculate_checksum($str) {
+        $checksum = 0;
+        $checklength = strlen($str);
+        for ($i = 0; $i < $checklength; $i++) {
+            $checksum += ord($str[$i]);
+        }
+        return $checksum;
+    }
+
+    /**
+     * Based on an OS path, adds either that path (if it's a file) or
+     * all its children (if it's a directory) into the list of files to
+     * archive.
+     *
+     * If a progress indicator is supplied and if this corresponds to a
+     * directory, then it will be repeatedly called with the same values. This
+     * allows the progress handler to respond in some way to avoid timeouts
+     * if required.
+     *
+     * @param array $expandedfiles List of all files to archive (output)
+     * @param string $archivepath Current path within archive
+     * @param string $path OS path on disk
+     * @param file_progress|null $progress Progress indicator or null if none
+     * @param int $done Value for progress indicator
+     * @return bool True if successful
+     */
+    protected function list_files_path(array &$expandedfiles, $archivepath, $path,
+            ?file_progress $progress , $done) {
+        if (is_dir($path)) {
+            // Unless we're using this directory as archive root, add a
+            // directory entry.
+            if ($archivepath != '') {
+                // Add directory-creation record.
+                $expandedfiles[$archivepath . '/'] = null;
+            }
+
+            // Loop through directory contents and recurse.
+            if (!$handle = opendir($path)) {
+                return false;
+            }
+            while (false !== ($entry = readdir($handle))) {
+                if ($entry === '.' || $entry === '..') {
+                    continue;
+                }
+                $result = $this->list_files_path($expandedfiles,
+                        $archivepath . '/' . $entry, $path . '/' . $entry,
+                        $progress, $done);
+                if (!$result) {
+                    return false;
+                }
+                if ($progress) {
+                    $progress->progress($done, self::PROGRESS_MAX);
+                }
+            }
+            closedir($handle);
+        } else {
+            // Just add it to list.
+            $expandedfiles[$archivepath] = $path;
+        }
+        return true;
+    }
+
+    /**
+     * Based on a stored_file objects, adds either that file (if it's a file) or
+     * all its children (if it's a directory) into the list of files to
+     * archive.
+     *
+     * If a progress indicator is supplied and if this corresponds to a
+     * directory, then it will be repeatedly called with the same values. This
+     * allows the progress handler to respond in some way to avoid timeouts
+     * if required.
+     *
+     * @param array $expandedfiles List of all files to archive (output)
+     * @param string $archivepath Current path within archive
+     * @param stored_file $file File object
+     */
+    protected function list_files_stored(array &$expandedfiles, $archivepath, stored_file $file) {
+        if ($file->is_directory()) {
+            // Add a directory-creation record.
+            $expandedfiles[$archivepath . '/'] = null;
+
+            // Loop through directory contents (this is a recursive collection
+            // of all children not just one directory).
+            $fs = get_file_storage();
+            $baselength = strlen($file->get_filepath());
+            $files = $fs->get_directory_files(
+                    $file->get_contextid(), $file->get_component(), $file->get_filearea(), $file->get_itemid(),
+                    $file->get_filepath(), true, true);
+            foreach ($files as $childfile) {
+                // Get full pathname after original part.
+                $path = $childfile->get_filepath();
+                $path = substr($path, $baselength);
+                $path = $archivepath . '/' . $path;
+                if ($childfile->is_directory()) {
+                    $childfile = null;
+                } else {
+                    $path .= $childfile->get_filename();
+                }
+                $expandedfiles[$path] = $childfile;
+            }
+        } else {
+            // Just add it to list.
+            $expandedfiles[$archivepath] = $file;
+        }
+    }
+
+    /**
+     * Extract file to given file path (real OS filesystem), existing files are overwritten.
+     *
+     * @param stored_file|string $archivefile full pathname of zip file or stored_file instance
+     * @param string $pathname target directory
+     * @param array $onlyfiles only extract files present in the array
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @param bool $returnbool Whether to return a basic true/false indicating error state, or full per-file error
+     * details.
+     * @return array list of processed files (name=>true)
+     * @throws moodle_exception If error
+     */
+    public function extract_to_pathname($archivefile, $pathname,
+            array $onlyfiles = null, file_progress $progress = null, $returnbool = false) {
+        $extractor = new tgz_extractor($archivefile);
+        try {
+            $result = $extractor->extract(
+                    new tgz_packer_extract_to_pathname($pathname, $onlyfiles), $progress);
+            if ($returnbool) {
+                if (!is_array($result)) {
+                    return false;
+                }
+                foreach ($result as $status) {
+                    if ($status !== true) {
+                        return false;
+                    }
+                }
+                return true;
+            } else {
+                return $result;
+            }
+        } catch (moodle_exception $e) {
+            if ($returnbool) {
+                return false;
+            } else {
+                throw $e;
+            }
+        }
+    }
+
+    /**
+     * Extract file to given file path (real OS filesystem), existing files are overwritten.
+     *
+     * @param string|stored_file $archivefile full pathname of zip file or stored_file instance
+     * @param int $contextid context ID
+     * @param string $component component
+     * @param string $filearea file area
+     * @param int $itemid item ID
+     * @param string $pathbase file path
+     * @param int $userid user ID
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return array list of processed files (name=>true)
+     * @throws moodle_exception If error
+     */
+    public function extract_to_storage($archivefile, $contextid,
+            $component, $filearea, $itemid, $pathbase, $userid = null,
+            file_progress $progress = null) {
+        $extractor = new tgz_extractor($archivefile);
+        return $extractor->extract(
+                new tgz_packer_extract_to_storage($contextid, $component,
+                    $filearea, $itemid, $pathbase, $userid), $progress);
+    }
+
+    /**
+     * Returns array of info about all files in archive.
+     *
+     * @param string|stored_file $archivefile
+     * @return array of file infos
+     */
+    public function list_files($archivefile) {
+        $extractor = new tgz_extractor($archivefile);
+        return $extractor->list_files();
+    }
+
+    /**
+     * Checks whether a file appears to be a .tar.gz file.
+     *
+     * @param string|stored_file $archivefile
+     * @return bool True if file contains the gzip magic number
+     */
+    public static function is_tgz_file($archivefile) {
+        if (is_a($archivefile, 'stored_file')) {
+            $fp = $archivefile->get_content_file_handle();
+        } else {
+            $fp = fopen($archivefile, 'rb');
+        }
+        $firstbytes = fread($fp, 2);
+        fclose($fp);
+        return ($firstbytes[0] == "\x1f" && $firstbytes[1] == "\x8b");
+    }
+
+    /**
+     * The zlib extension is required for this packer to work. This is a single
+     * location for the code to check whether the extension is available.
+     *
+     * @deprecated since 2.7 Always true because zlib extension is now required.
+     *
+     * @return bool True if the zlib extension is available OK
+     */
+    public static function has_required_extension() {
+        return extension_loaded('zlib');
+    }
+}
+
+
+/**
+ * Handles extraction to pathname.
+ */
+class tgz_packer_extract_to_pathname implements tgz_extractor_handler {
+    /**
+     * @var string Target directory for extract.
+     */
+    protected $pathname;
+    /**
+     * @var array Array of files to extract (other files are skipped).
+     */
+    protected $onlyfiles;
+
+    /**
+     * Constructor.
+     *
+     * @param string $pathname target directory
+     * @param array $onlyfiles only extract files present in the array
+     */
+    public function __construct($pathname, array $onlyfiles = null) {
+        $this->pathname = $pathname;
+        $this->onlyfiles = $onlyfiles;
+    }
+
+    /**
+     * @see tgz_extractor_handler::tgz_start_file()
+     */
+    public function tgz_start_file($archivepath) {
+        // Check file restriction.
+        if ($this->onlyfiles !== null && !in_array($archivepath, $this->onlyfiles)) {
+            return null;
+        }
+        // Ensure directory exists and prepare filename.
+        $fullpath = $this->pathname . '/' . $archivepath;
+        check_dir_exists(dirname($fullpath));
+        return $fullpath;
+    }
+
+    /**
+     * @see tgz_extractor_handler::tgz_end_file()
+     */
+    public function tgz_end_file($archivepath, $realpath) {
+        // Do nothing.
+    }
+
+    /**
+     * @see tgz_extractor_handler::tgz_directory()
+     */
+    public function tgz_directory($archivepath, $mtime) {
+        // Check file restriction.
+        if ($this->onlyfiles !== null && !in_array($archivepath, $this->onlyfiles)) {
+            return false;
+        }
+        // Ensure directory exists.
+        $fullpath = $this->pathname . '/' . $archivepath;
+        check_dir_exists($fullpath);
+        return true;
+    }
+}
+
+
+/**
+ * Handles extraction to file storage.
+ */
+class tgz_packer_extract_to_storage implements tgz_extractor_handler {
+    /**
+     * @var string Path to temp file.
+     */
+    protected $tempfile;
+
+    /**
+     * @var int Context id for files.
+     */
+    protected $contextid;
+    /**
+     * @var string Component name for files.
+     */
+    protected $component;
+    /**
+     * @var string File area for files.
+     */
+    protected $filearea;
+    /**
+     * @var int Item ID for files.
+     */
+    protected $itemid;
+    /**
+     * @var string Base path for files (subfolders will go inside this).
+     */
+    protected $pathbase;
+    /**
+     * @var int User id for files or null if none.
+     */
+    protected $userid;
+
+    /**
+     * Constructor.
+     *
+     * @param int $contextid Context id for files.
+     * @param string $component Component name for files.
+     * @param string $filearea File area for files.
+     * @param int $itemid Item ID for files.
+     * @param string $pathbase Base path for files (subfolders will go inside this).
+     * @param int $userid User id for files or null if none.
+     */
+    public function __construct($contextid, $component, $filearea, $itemid, $pathbase, $userid) {
+        global $CFG;
+
+        // Store all data.
+        $this->contextid = $contextid;
+        $this->component = $component;
+        $this->filearea = $filearea;
+        $this->itemid = $itemid;
+        $this->pathbase = $pathbase;
+        $this->userid = $userid;
+
+        // Obtain temp filename.
+        $tempfolder = $CFG->tempdir . '/core_files';
+        check_dir_exists($tempfolder);
+        $this->tempfile = tempnam($tempfolder, '.dat');
+    }
+
+    /**
+     * @see tgz_extractor_handler::tgz_start_file()
+     */
+    public function tgz_start_file($archivepath) {
+        // All files are stored in the same filename.
+        return $this->tempfile;
+    }
+
+    /**
+     * @see tgz_extractor_handler::tgz_end_file()
+     */
+    public function tgz_end_file($archivepath, $realpath) {
+        // Place temp file into storage.
+        $fs = get_file_storage();
+        $filerecord = array('contextid' => $this->contextid, 'component' => $this->component,
+                'filearea' => $this->filearea, 'itemid' => $this->itemid);
+        $filerecord['filepath'] = $this->pathbase . dirname($archivepath) . '/';
+        $filerecord['filename'] = basename($archivepath);
+        if ($this->userid) {
+            $filerecord['userid'] = $this->userid;
+        }
+        // Delete existing file (if any) and create new one.
+        tgz_packer::delete_existing_file_record($fs, $filerecord);
+        $fs->create_file_from_pathname($filerecord, $this->tempfile);
+        unlink($this->tempfile);
+    }
+
+    /**
+     * @see tgz_extractor_handler::tgz_directory()
+     */
+    public function tgz_directory($archivepath, $mtime) {
+        // Standardise path.
+        if (!preg_match('~/$~', $archivepath)) {
+            $archivepath .= '/';
+        }
+        // Create directory if it doesn't already exist.
+        $fs = get_file_storage();
+        if (!$fs->file_exists($this->contextid, $this->component, $this->filearea, $this->itemid,
+                $this->pathbase . $archivepath, '.')) {
+            $fs->create_directory($this->contextid, $this->component, $this->filearea, $this->itemid,
+                    $this->pathbase . $archivepath);
+        }
+        return true;
+    }
+}

lib/filestorage/zip_archive.php

@@ -0,0 +1,893 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Implementation of zip file archive.
+ *
+ * @package   core_files
+ * @copyright 2008 Petr Skoda (http://skodak.org)
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+require_once("$CFG->libdir/filestorage/file_archive.php");
+
+/**
+ * Zip file archive class.
+ *
+ * @package   core_files
+ * @category  files
+ * @copyright 2008 Petr Skoda (http://skodak.org)
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class zip_archive extends file_archive {
+
+    /** @var string Pathname of archive */
+    protected $archivepathname = null;
+
+    /** @var int archive open mode */
+    protected $mode = null;
+
+    /** @var int Used memory tracking */
+    protected $usedmem = 0;
+
+    /** @var int Iteration position */
+    protected $pos = 0;
+
+    /** @var ZipArchive instance */
+    protected $za;
+
+    /** @var bool was this archive modified? */
+    protected $modified = false;
+
+    /** @var array unicode decoding array, created by decoding zip file */
+    protected $namelookup = null;
+
+    /** @var string base64 encoded contents of empty zip file */
+    protected static $emptyzipcontent = 'UEsFBgAAAAAAAAAAAAAAAAAAAAAAAA==';
+
+    /** @var bool ugly hack for broken empty zip handling in < PHP 5.3.10 */
+    protected $emptyziphack = false;
+
+    /**
+     * Create new zip_archive instance.
+     */
+    public function __construct() {
+        $this->encoding = null; // Autodetects encoding by default.
+    }
+
+    /**
+     * Open or create archive (depending on $mode).
+     *
+     * @todo MDL-31048 return error message
+     * @param string $archivepathname
+     * @param int $mode OPEN, CREATE or OVERWRITE constant
+     * @param string $encoding archive local paths encoding, empty means autodetect
+     * @return bool success
+     */
+    public function open($archivepathname, $mode=file_archive::CREATE, $encoding=null) {
+        $this->close();
+
+        $this->usedmem  = 0;
+        $this->pos      = 0;
+        $this->encoding = $encoding;
+        $this->mode     = $mode;
+
+        $this->za = new ZipArchive();
+
+        switch($mode) {
+            case file_archive::OPEN:      $flags = 0; break;
+            case file_archive::OVERWRITE: $flags = ZIPARCHIVE::CREATE | ZIPARCHIVE::OVERWRITE; break; //changed in PHP 5.2.8
+            case file_archive::CREATE:
+            default :                     $flags = ZIPARCHIVE::CREATE; break;
+        }
+
+        $result = $this->za->open($archivepathname, $flags);
+
+        if ($flags == 0 and $result === ZIPARCHIVE::ER_NOZIP and filesize($archivepathname) === 22) {
+            // Legacy PHP versions < 5.3.10 can not deal with empty zip archives.
+            if (file_get_contents($archivepathname) === base64_decode(self::$emptyzipcontent)) {
+                if ($temp = make_temp_directory('zip')) {
+                    $this->emptyziphack = tempnam($temp, 'zip');
+                    $this->za = new ZipArchive();
+                    $result = $this->za->open($this->emptyziphack, ZIPARCHIVE::CREATE);
+                }
+            }
+        }
+
+        if ($result === true) {
+            if (file_exists($archivepathname)) {
+                $this->archivepathname = realpath($archivepathname);
+            } else {
+                $this->archivepathname = $archivepathname;
+            }
+            return true;
+
+        } else {
+            $message = 'Unknown error.';
+            switch ($result) {
+                case ZIPARCHIVE::ER_EXISTS: $message = 'File already exists.'; break;
+                case ZIPARCHIVE::ER_INCONS: $message = 'Zip archive inconsistent.'; break;
+                case ZIPARCHIVE::ER_INVAL: $message = 'Invalid argument.'; break;
+                case ZIPARCHIVE::ER_MEMORY: $message = 'Malloc failure.'; break;
+                case ZIPARCHIVE::ER_NOENT: $message = 'No such file.'; break;
+                case ZIPARCHIVE::ER_NOZIP: $message = 'Not a zip archive.'; break;
+                case ZIPARCHIVE::ER_OPEN: $message = 'Can\'t open file.'; break;
+                case ZIPARCHIVE::ER_READ: $message = 'Read error.'; break;
+                case ZIPARCHIVE::ER_SEEK: $message = 'Seek error.'; break;
+            }
+            debugging($message.': '.$archivepathname, DEBUG_DEVELOPER);
+            $this->za = null;
+            $this->archivepathname = null;
+            return false;
+        }
+    }
+
+    /**
+     * Normalize $localname, always keep in utf-8 encoding.
+     *
+     * @param string $localname name of file in utf-8 encoding
+     * @return string normalised compressed file or directory name
+     */
+    protected function mangle_pathname($localname) {
+        $result = str_replace('\\', '/', $localname);   // no MS \ separators
+        $result = preg_replace('/\.\.+\//', '', $result); // Cleanup any potential ../ transversal (any number of dots).
+        $result = preg_replace('/\.\.+/', '.', $result); // Join together any number of consecutive dots.
+        $result = ltrim($result, '/');                  // no leading slash
+
+        if ($result === '.') {
+            $result = '';
+        }
+
+        return $result;
+    }
+
+    /**
+     * Tries to convert $localname into utf-8
+     * please note that it may fail really badly.
+     * The resulting file name is cleaned.
+     *
+     * @param string $localname name (encoding is read from zip file or guessed)
+     * @return string in utf-8
+     */
+    protected function unmangle_pathname($localname) {
+        $this->init_namelookup();
+
+        if (!isset($this->namelookup[$localname])) {
+            $name = $localname;
+            // This should not happen.
+            if (!empty($this->encoding) and $this->encoding !== 'utf-8') {
+                $name = @core_text::convert($name, $this->encoding, 'utf-8');
+            }
+            $name = str_replace('\\', '/', $name);   // no MS \ separators
+            $name = clean_param($name, PARAM_PATH);  // only safe chars
+            return ltrim($name, '/');                // no leading slash
+        }
+
+        return $this->namelookup[$localname];
+    }
+
+    /**
+     * Close archive, write changes to disk.
+     *
+     * @return bool success
+     */
+    public function close() {
+        if (!isset($this->za)) {
+            return false;
+        }
+
+        if ($this->emptyziphack) {
+            @$this->za->close();
+            $this->za = null;
+            $this->mode = null;
+            $this->namelookup = null;
+            $this->modified = false;
+            @unlink($this->emptyziphack);
+            $this->emptyziphack = false;
+            return true;
+
+        } else if ($this->za->numFiles == 0) {
+            // PHP can not create empty archives, so let's fake it.
+            @$this->za->close();
+            $this->za = null;
+            $this->mode = null;
+            $this->namelookup = null;
+            $this->modified = false;
+            // If the existing archive is already empty, we didn't change it.  Don't bother completing a save.
+            // This is important when we are inspecting archives that we might not have write permission to.
+            if (@filesize($this->archivepathname) == 22 &&
+                    @file_get_contents($this->archivepathname) === base64_decode(self::$emptyzipcontent)) {
+                return true;
+            }
+            @unlink($this->archivepathname);
+            $data = base64_decode(self::$emptyzipcontent);
+            if (!file_put_contents($this->archivepathname, $data)) {
+                return false;
+            }
+            return true;
+        }
+
+        $res = $this->za->close();
+        $this->za = null;
+        $this->mode = null;
+        $this->namelookup = null;
+
+        if ($this->modified) {
+            $this->fix_utf8_flags();
+            $this->modified = false;
+        }
+
+        return $res;
+    }
+
+    /**
+     * Returns file stream for reading of content.
+     *
+     * @param int $index index of file
+     * @return resource|bool file handle or false if error
+     */
+    public function get_stream($index) {
+        if (!isset($this->za)) {
+            return false;
+        }
+
+        $name = $this->za->getNameIndex($index);
+        if ($name === false) {
+            return false;
+        }
+
+        return $this->za->getStream($name);
+    }
+
+    /**
+     * Extract the archive contents to the given location.
+     *
+     * @param string $destination Path to the location where to extract the files.
+     * @param int $index Index of the archive entry.
+     * @return bool true on success or false on failure
+     */
+    public function extract_to($destination, $index) {
+
+        if (!isset($this->za)) {
+            return false;
+        }
+
+        $name = $this->za->getNameIndex($index);
+
+        if ($name === false) {
+            return false;
+        }
+
+        return $this->za->extractTo($destination, $name);
+    }
+
+    /**
+     * Returns file information.
+     *
+     * @param int $index index of file
+     * @return stdClass|bool info object or false if error
+     */
+    public function get_info($index) {
+        if (!isset($this->za)) {
+            return false;
+        }
+
+        // Need to use the ZipArchive's numfiles, as $this->count() relies on this function to count actual files (skipping OSX junk).
+        if ($index < 0 or $index >=$this->za->numFiles) {
+            return false;
+        }
+
+        // PHP 5.6 introduced encoding guessing logic for file names. To keep consistent behaviour with older versions,
+        // we fall back to obtaining file names as raw unmodified strings.
+        $result = $this->za->statIndex($index, ZipArchive::FL_ENC_RAW);
+
+        if ($result === false) {
+            return false;
+        }
+
+        $info = new stdClass();
+        $info->index             = $index;
+        $info->original_pathname = $result['name'];
+        $info->pathname          = $this->unmangle_pathname($result['name']);
+        $info->mtime             = (int)$result['mtime'];
+
+        if ($info->pathname[strlen($info->pathname)-1] === '/') {
+            $info->is_directory = true;
+            $info->size         = 0;
+        } else {
+            $info->is_directory = false;
+            $info->size         = (int)$result['size'];
+        }
+
+        if ($this->is_system_file($info)) {
+            // Don't return system files.
+            return false;
+        }
+
+        return $info;
+    }
+
+    /**
+     * Returns array of info about all files in archive.
+     *
+     * @return array of file infos
+     */
+    public function list_files() {
+        if (!isset($this->za)) {
+            return false;
+        }
+
+        $infos = array();
+
+        foreach ($this as $info) {
+            // Simply iterating over $this will give us info only for files we're interested in.
+            array_push($infos, $info);
+        }
+
+        return $infos;
+    }
+
+    public function is_system_file($fileinfo) {
+        if (substr($fileinfo->pathname, 0, 8) === '__MACOSX' or substr($fileinfo->pathname, -9) === '.DS_Store') {
+            // Mac OSX system files.
+            return true;
+        }
+        if (substr($fileinfo->pathname, -9) === 'Thumbs.db') {
+            $stream = $this->za->getStream($fileinfo->pathname);
+            $info = base64_encode(fread($stream, 8));
+            fclose($stream);
+            if ($info === '0M8R4KGxGuE=') {
+                // It's an OLE Compound File - so it's almost certainly a Windows thumbnail cache.
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Returns number of files in archive.
+     *
+     * @return int number of files
+     */
+    public function count(): int {
+        if (!isset($this->za)) {
+            return false;
+        }
+
+        return count($this->list_files());
+    }
+
+    /**
+     * Returns approximate number of files in archive. This may be a slight
+     * overestimate.
+     *
+     * @return int|bool Estimated number of files, or false if not opened
+     */
+    public function estimated_count() {
+        if (!isset($this->za)) {
+            return false;
+        }
+
+        return $this->za->numFiles;
+    }
+
+    /**
+     * Add file into archive.
+     *
+     * @param string $localname name of file in archive
+     * @param string $pathname location of file
+     * @return bool success
+     */
+    public function add_file_from_pathname($localname, $pathname) {
+        if ($this->emptyziphack) {
+            $this->close();
+            $this->open($this->archivepathname, file_archive::OVERWRITE, $this->encoding);
+        }
+
+        if (!isset($this->za)) {
+            return false;
+        }
+
+        if ($this->archivepathname === realpath($pathname)) {
+            // Do not add self into archive.
+            return false;
+        }
+
+        if (!is_readable($pathname) or is_dir($pathname)) {
+            return false;
+        }
+
+        if (is_null($localname)) {
+            $localname = clean_param($pathname, PARAM_PATH);
+        }
+        $localname = trim($localname, '/'); // No leading slashes in archives!
+        $localname = $this->mangle_pathname($localname);
+
+        if ($localname === '') {
+            // Sorry - conversion failed badly.
+            return false;
+        }
+
+        if (!$this->za->addFile($pathname, $localname)) {
+            return false;
+        }
+        $this->modified = true;
+        return true;
+    }
+
+    /**
+     * Add content of string into archive.
+     *
+     * @param string $localname name of file in archive
+     * @param string $contents contents
+     * @return bool success
+     */
+    public function add_file_from_string($localname, $contents) {
+        if ($this->emptyziphack) {
+            $this->close();
+            $this->open($this->archivepathname, file_archive::OVERWRITE, $this->encoding);
+        }
+
+        if (!isset($this->za)) {
+            return false;
+        }
+
+        $localname = trim($localname, '/'); // No leading slashes in archives!
+        $localname = $this->mangle_pathname($localname);
+
+        if ($localname === '') {
+            // Sorry - conversion failed badly.
+            return false;
+        }
+
+        if ($this->usedmem > 2097151) {
+            // This prevents running out of memory when adding many large files using strings.
+            $this->close();
+            $res = $this->open($this->archivepathname, file_archive::OPEN, $this->encoding);
+            if ($res !== true) {
+                throw new \moodle_exception('cannotopenzip');
+            }
+        }
+        $this->usedmem += strlen($contents);
+
+        if (!$this->za->addFromString($localname, $contents)) {
+            return false;
+        }
+        $this->modified = true;
+        return true;
+    }
+
+    /**
+     * Add empty directory into archive.
+     *
+     * @param string $localname name of file in archive
+     * @return bool success
+     */
+    public function add_directory($localname) {
+        if ($this->emptyziphack) {
+            $this->close();
+            $this->open($this->archivepathname, file_archive::OVERWRITE, $this->encoding);
+        }
+
+        if (!isset($this->za)) {
+            return false;
+        }
+        $localname = trim($localname, '/'). '/';
+        $localname = $this->mangle_pathname($localname);
+
+        if ($localname === '/') {
+            // Sorry - conversion failed badly.
+            return false;
+        }
+
+        if ($localname !== '') {
+            if (!$this->za->addEmptyDir($localname)) {
+                return false;
+            }
+            $this->modified = true;
+        }
+        return true;
+    }
+
+    /**
+     * Returns current file info.
+     *
+     * @return stdClass
+     */
+    #[\ReturnTypeWillChange]
+    public function current() {
+        if (!isset($this->za)) {
+            return false;
+        }
+
+        return $this->get_info($this->pos);
+    }
+
+    /**
+     * Returns the index of current file.
+     *
+     * @return int current file index
+     */
+    #[\ReturnTypeWillChange]
+    public function key() {
+        return $this->pos;
+    }
+
+    /**
+     * Moves forward to next file.
+     */
+    public function next(): void {
+        $this->pos++;
+    }
+
+    /**
+     * Rewinds back to the first file.
+     */
+    public function rewind(): void {
+        $this->pos = 0;
+    }
+
+    /**
+     * Did we reach the end?
+     *
+     * @return bool
+     */
+    public function valid(): bool {
+        if (!isset($this->za)) {
+            return false;
+        }
+
+        // Skip over unwanted system files (get_info will return false).
+        while (!$this->get_info($this->pos) && $this->pos < $this->za->numFiles) {
+            $this->next();
+        }
+
+        // No files left - we're at the end.
+        if ($this->pos >= $this->za->numFiles) {
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * Create a map of file names used in zip archive.
+     * @return void
+     */
+    protected function init_namelookup() {
+        if ($this->emptyziphack) {
+            $this->namelookup = array();
+            return;
+        }
+
+        if (!isset($this->za)) {
+            return;
+        }
+        if (isset($this->namelookup)) {
+            return;
+        }
+
+        $this->namelookup = array();
+
+        if ($this->mode != file_archive::OPEN) {
+            // No need to tweak existing names when creating zip file because there are none yet!
+            return;
+        }
+
+        if (!file_exists($this->archivepathname)) {
+            return;
+        }
+
+        if (!$fp = fopen($this->archivepathname, 'rb')) {
+            return;
+        }
+        if (!$filesize = filesize($this->archivepathname)) {
+            return;
+        }
+
+        $centralend = self::zip_get_central_end($fp, $filesize);
+
+        if ($centralend === false or $centralend['disk'] !== 0 or $centralend['disk_start'] !== 0 or $centralend['offset'] === 0xFFFFFFFF) {
+            // Single disk archives only and o support for ZIP64, sorry.
+            fclose($fp);
+            return;
+        }
+
+        fseek($fp, $centralend['offset']);
+        $data = fread($fp, $centralend['size']);
+        $pos = 0;
+        $files = array();
+        for($i=0; $i<$centralend['entries']; $i++) {
+            $file = self::zip_parse_file_header($data, $centralend, $pos);
+            if ($file === false) {
+                // Wrong header, sorry.
+                fclose($fp);
+                return;
+            }
+            $files[] = $file;
+        }
+        fclose($fp);
+
+        foreach ($files as $file) {
+            $name = $file['name'];
+            if (preg_match('/^[a-zA-Z0-9_\-\.]*$/', $file['name'])) {
+                // No need to fix ASCII.
+                $name = fix_utf8($name);
+
+            } else if (!($file['general'] & pow(2, 11))) {
+                // First look for unicode name alternatives.
+                $found = false;
+                foreach($file['extra'] as $extra) {
+                    if ($extra['id'] === 0x7075) {
+                        $data = unpack('cversion/Vcrc', substr($extra['data'], 0, 5));
+                        if ($data['crc'] === crc32($name)) {
+                            $found = true;
+                            $name = substr($extra['data'], 5);
+                        }
+                    }
+                }
+                if (!$found and !empty($this->encoding) and $this->encoding !== 'utf-8') {
+                    // Try the encoding from open().
+                    $newname = @core_text::convert($name, $this->encoding, 'utf-8');
+                    $original  = core_text::convert($newname, 'utf-8', $this->encoding);
+                    if ($original === $name) {
+                        $found = true;
+                        $name = $newname;
+                    }
+                }
+                if (!$found and $file['version'] === 0x315) {
+                    // This looks like OS X build in zipper.
+                    $newname = fix_utf8($name);
+                    if ($newname === $name) {
+                        $found = true;
+                        $name = $newname;
+                    }
+                }
+                if (!$found and $file['version'] === 0) {
+                    // This looks like our old borked Moodle 2.2 file.
+                    $newname = fix_utf8($name);
+                    if ($newname === $name) {
+                        $found = true;
+                        $name = $newname;
+                    }
+                }
+                if (!$found and $encoding = get_string('oldcharset', 'langconfig')) {
+                    // Last attempt - try the dos/unix encoding from current language.
+                    $windows = true;
+                    foreach($file['extra'] as $extra) {
+                        // In Windows archivers do not usually set any extras with the exception of NTFS flag in WinZip/WinRar.
+                        $windows = false;
+                        if ($extra['id'] === 0x000a) {
+                            $windows = true;
+                            break;
+                        }
+                    }
+
+                    if ($windows === true) {
+                        switch(strtoupper($encoding)) {
+                            case 'ISO-8859-1': $encoding = 'CP850'; break;
+                            case 'ISO-8859-2': $encoding = 'CP852'; break;
+                            case 'ISO-8859-4': $encoding = 'CP775'; break;
+                            case 'ISO-8859-5': $encoding = 'CP866'; break;
+                            case 'ISO-8859-6': $encoding = 'CP720'; break;
+                            case 'ISO-8859-7': $encoding = 'CP737'; break;
+                            case 'ISO-8859-8': $encoding = 'CP862'; break;
+                            case 'WINDOWS-1251': $encoding = 'CP866'; break;
+                            case 'EUC-JP':
+                            case 'UTF-8':
+                                if ($winchar = get_string('localewincharset', 'langconfig')) {
+                                    // Most probably works only for zh_cn,
+                                    // if there are more problems we could add zipcharset to langconfig files.
+                                    $encoding = $winchar;
+                                }
+                                break;
+                        }
+                    }
+                    $newname = @core_text::convert($name, $encoding, 'utf-8');
+                    $original  = core_text::convert($newname, 'utf-8', $encoding);
+
+                    if ($original === $name) {
+                        $name = $newname;
+                    }
+                }
+            }
+            $name = str_replace('\\', '/', $name);  // no MS \ separators
+            $name = clean_param($name, PARAM_PATH); // only safe chars
+            $name = ltrim($name, '/');              // no leading slash
+
+            if (function_exists('normalizer_normalize')) {
+                $name = normalizer_normalize($name, Normalizer::FORM_C);
+            }
+
+            $this->namelookup[$file['name']] = $name;
+        }
+    }
+
+    /**
+     * Add unicode flag to all files in archive.
+     *
+     * NOTE: single disk archives only, no ZIP64 support.
+     *
+     * @return bool success, modifies the file contents
+     */
+    protected function fix_utf8_flags() {
+        if ($this->emptyziphack) {
+            return true;
+        }
+
+        if (!file_exists($this->archivepathname)) {
+            return true;
+        }
+
+        // Note: the ZIP structure is described at http://www.pkware.com/documents/casestudies/APPNOTE.TXT
+        if (!$fp = fopen($this->archivepathname, 'rb+')) {
+            return false;
+        }
+        if (!$filesize = filesize($this->archivepathname)) {
+            return false;
+        }
+
+        $centralend = self::zip_get_central_end($fp, $filesize);
+
+        if ($centralend === false or $centralend['disk'] !== 0 or $centralend['disk_start'] !== 0 or $centralend['offset'] === 0xFFFFFFFF) {
+            // Single disk archives only and o support for ZIP64, sorry.
+            fclose($fp);
+            return false;
+        }
+
+        fseek($fp, $centralend['offset']);
+        $data = fread($fp, $centralend['size']);
+        $pos = 0;
+        $files = array();
+        for($i=0; $i<$centralend['entries']; $i++) {
+            $file = self::zip_parse_file_header($data, $centralend, $pos);
+            if ($file === false) {
+                // Wrong header, sorry.
+                fclose($fp);
+                return false;
+            }
+
+            $newgeneral = $file['general'] | pow(2, 11);
+            if ($newgeneral === $file['general']) {
+                // Nothing to do with this file.
+                continue;
+            }
+
+            if (preg_match('/^[a-zA-Z0-9_\-\.]*$/', $file['name'])) {
+                // ASCII file names are always ok.
+                continue;
+            }
+            if ($file['extra']) {
+                // Most probably not created by php zip ext, better to skip it.
+                continue;
+            }
+            if (fix_utf8($file['name']) !== $file['name']) {
+                // Does not look like a valid utf-8 encoded file name, skip it.
+                continue;
+            }
+
+            // Read local file header.
+            fseek($fp, $file['local_offset']);
+            $localfile = unpack('Vsig/vversion_req/vgeneral/vmethod/vmtime/vmdate/Vcrc/Vsize_compressed/Vsize/vname_length/vextra_length', fread($fp, 30));
+            if ($localfile['sig'] !== 0x04034b50) {
+                // Borked file!
+                fclose($fp);
+                return false;
+            }
+
+            $file['local'] = $localfile;
+            $files[] = $file;
+        }
+
+        foreach ($files as $file) {
+            $localfile = $file['local'];
+            // Add the unicode flag in central file header.
+            fseek($fp, $file['central_offset'] + 8);
+            if (ftell($fp) === $file['central_offset'] + 8) {
+                $newgeneral = $file['general'] | pow(2, 11);
+                fwrite($fp, pack('v', $newgeneral));
+            }
+            // Modify local file header too.
+            fseek($fp, $file['local_offset'] + 6);
+            if (ftell($fp) === $file['local_offset'] + 6) {
+                $newgeneral = $localfile['general'] | pow(2, 11);
+                fwrite($fp, pack('v', $newgeneral));
+            }
+        }
+
+        fclose($fp);
+        return true;
+    }
+
+    /**
+     * Read end of central signature of ZIP file.
+     * @internal
+     * @static
+     * @param resource $fp
+     * @param int $filesize
+     * @return array|bool
+     */
+    public static function zip_get_central_end($fp, $filesize) {
+        // Find end of central directory record.
+        fseek($fp, $filesize - 22);
+        $info = unpack('Vsig', fread($fp, 4));
+        if ($info['sig'] === 0x06054b50) {
+            // There is no comment.
+            fseek($fp, $filesize - 22);
+            $data = fread($fp, 22);
+        } else {
+            // There is some comment with 0xFF max size - that is 65557.
+            fseek($fp, $filesize - 65557);
+            $data = fread($fp, 65557);
+        }
+
+        $pos = strpos($data, pack('V', 0x06054b50));
+        if ($pos === false) {
+            // Borked ZIP structure!
+            return false;
+        }
+        $centralend = unpack('Vsig/vdisk/vdisk_start/vdisk_entries/ventries/Vsize/Voffset/vcomment_length', substr($data, $pos, 22));
+        if ($centralend['comment_length']) {
+            $centralend['comment'] = substr($data, 22, $centralend['comment_length']);
+        } else {
+            $centralend['comment'] = '';
+        }
+
+        return $centralend;
+    }
+
+    /**
+     * Parse file header.
+     * @internal
+     * @param string $data
+     * @param array $centralend
+     * @param int $pos (modified)
+     * @return array|bool file info
+     */
+    public static function zip_parse_file_header($data, $centralend, &$pos) {
+        $file = unpack('Vsig/vversion/vversion_req/vgeneral/vmethod/Vmodified/Vcrc/Vsize_compressed/Vsize/vname_length/vextra_length/vcomment_length/vdisk/vattr/Vattrext/Vlocal_offset', substr($data, $pos, 46));
+        $file['central_offset'] = $centralend['offset'] + $pos;
+        $pos = $pos + 46;
+        if ($file['sig'] !== 0x02014b50) {
+            // Borked ZIP structure!
+            return false;
+        }
+        $file['name'] = substr($data, $pos, $file['name_length']);
+        $pos = $pos + $file['name_length'];
+        $file['extra'] = array();
+        $file['extra_data'] = '';
+        if ($file['extra_length']) {
+            $extradata = substr($data, $pos, $file['extra_length']);
+            $file['extra_data'] = $extradata;
+            while (strlen($extradata) > 4) {
+                $extra = unpack('vid/vsize', substr($extradata, 0, 4));
+                $extra['data'] = substr($extradata, 4, $extra['size']);
+                $extradata = substr($extradata, 4+$extra['size']);
+                $file['extra'][] = $extra;
+            }
+            $pos = $pos + $file['extra_length'];
+        }
+        if ($file['comment_length']) {
+            $pos = $pos + $file['comment_length'];
+            $file['comment'] = substr($data, $pos, $file['comment_length']);
+        } else {
+            $file['comment'] = '';
+        }
+        return $file;
+    }
+}

lib/filestorage/zip_packer.php

@@ -0,0 +1,613 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Implementation of zip packer.
+ *
+ * @package   core_files
+ * @copyright 2008 Petr Skoda (http://skodak.org)
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+require_once("$CFG->libdir/filestorage/file_packer.php");
+require_once("$CFG->libdir/filestorage/zip_archive.php");
+
+/**
+ * Utility class - handles all zipping and unzipping operations.
+ *
+ * @package   core_files
+ * @category  files
+ * @copyright 2008 Petr Skoda (http://skodak.org)
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class zip_packer extends file_packer {
+
+    /**
+     * Zip files and store the result in file storage.
+     *
+     * @param array $files array with full zip paths (including directory information)
+     *              as keys (archivepath=>ospathname or archivepath/subdir=>stored_file or archivepath=>array('content_as_string'))
+     * @param int $contextid context ID
+     * @param string $component component
+     * @param string $filearea file area
+     * @param int $itemid item ID
+     * @param string $filepath file path
+     * @param string $filename file name
+     * @param int $userid user ID
+     * @param bool $ignoreinvalidfiles true means ignore missing or invalid files, false means abort on any error
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return stored_file|bool false if error stored_file instance if ok
+     */
+    public function archive_to_storage(array $files, $contextid,
+            $component, $filearea, $itemid, $filepath, $filename,
+            $userid = NULL, $ignoreinvalidfiles=true, file_progress $progress = null) {
+        global $CFG;
+
+        $fs = get_file_storage();
+
+        check_dir_exists($CFG->tempdir.'/zip');
+        $tmpfile = tempnam($CFG->tempdir.'/zip', 'zipstor');
+
+        if ($result = $this->archive_to_pathname($files, $tmpfile, $ignoreinvalidfiles, $progress)) {
+            if ($file = $fs->get_file($contextid, $component, $filearea, $itemid, $filepath, $filename)) {
+                if (!$file->delete()) {
+                    @unlink($tmpfile);
+                    return false;
+                }
+            }
+            $file_record = new stdClass();
+            $file_record->contextid = $contextid;
+            $file_record->component = $component;
+            $file_record->filearea  = $filearea;
+            $file_record->itemid    = $itemid;
+            $file_record->filepath  = $filepath;
+            $file_record->filename  = $filename;
+            $file_record->userid    = $userid;
+            $file_record->mimetype  = 'application/zip';
+
+            $result = $fs->create_file_from_pathname($file_record, $tmpfile);
+        }
+        @unlink($tmpfile);
+        return $result;
+    }
+
+    /**
+     * Zip files and store the result in os file.
+     *
+     * @param array $files array with zip paths as keys (archivepath=>ospathname or archivepath=>stored_file or archivepath=>array('content_as_string'))
+     * @param string $archivefile path to target zip file
+     * @param bool $ignoreinvalidfiles true means ignore missing or invalid files, false means abort on any error
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return bool true if file created, false if not
+     */
+    public function archive_to_pathname(array $files, $archivefile,
+            $ignoreinvalidfiles=true, file_progress $progress = null) {
+        $ziparch = new zip_archive();
+        if (!$ziparch->open($archivefile, file_archive::OVERWRITE)) {
+            return false;
+        }
+
+        $abort = false;
+        foreach ($files as $archivepath => $file) {
+            $archivepath = trim($archivepath, '/');
+
+            // Record progress each time around this loop.
+            if ($progress) {
+                $progress->progress();
+            }
+
+            if (is_null($file)) {
+                // Directories have null as content.
+                if (!$ziparch->add_directory($archivepath.'/')) {
+                    debugging("Can not zip '$archivepath' directory", DEBUG_DEVELOPER);
+                    if (!$ignoreinvalidfiles) {
+                        $abort = true;
+                        break;
+                    }
+                }
+
+            } else if (is_string($file)) {
+                if (!$this->archive_pathname($ziparch, $archivepath, $file, $progress)) {
+                    debugging("Can not zip '$archivepath' file", DEBUG_DEVELOPER);
+                    if (!$ignoreinvalidfiles) {
+                        $abort = true;
+                        break;
+                    }
+                }
+
+            } else if (is_array($file)) {
+                $content = reset($file);
+                if (!$ziparch->add_file_from_string($archivepath, $content)) {
+                    debugging("Can not zip '$archivepath' file", DEBUG_DEVELOPER);
+                    if (!$ignoreinvalidfiles) {
+                        $abort = true;
+                        break;
+                    }
+                }
+
+            } else {
+                if (!$this->archive_stored($ziparch, $archivepath, $file, $progress)) {
+                    debugging("Can not zip '$archivepath' file", DEBUG_DEVELOPER);
+                    if (!$ignoreinvalidfiles) {
+                        $abort = true;
+                        break;
+                    }
+                }
+            }
+        }
+
+        if (!$ziparch->close()) {
+            @unlink($archivefile);
+            return false;
+        }
+
+        if ($abort) {
+            @unlink($archivefile);
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * Perform archiving file from stored file.
+     *
+     * @param zip_archive $ziparch zip archive instance
+     * @param string $archivepath file path to archive
+     * @param stored_file $file stored_file object
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return bool success
+     */
+    private function archive_stored($ziparch, $archivepath, $file, file_progress $progress = null) {
+        $result = $file->archive_file($ziparch, $archivepath);
+        if (!$result) {
+            return false;
+        }
+
+        if (!$file->is_directory()) {
+            return true;
+        }
+
+        $baselength = strlen($file->get_filepath());
+        $fs = get_file_storage();
+        $files = $fs->get_directory_files($file->get_contextid(), $file->get_component(), $file->get_filearea(), $file->get_itemid(),
+                                          $file->get_filepath(), true, true);
+        foreach ($files as $file) {
+            // Record progress for each file.
+            if ($progress) {
+                $progress->progress();
+            }
+
+            $path = $file->get_filepath();
+            $path = substr($path, $baselength);
+            $path = $archivepath.'/'.$path;
+            if (!$file->is_directory()) {
+                $path = $path.$file->get_filename();
+            }
+            // Ignore result here, partial zipping is ok for now.
+            $file->archive_file($ziparch, $path);
+        }
+
+        return true;
+    }
+
+    /**
+     * Perform archiving file from file path.
+     *
+     * @param zip_archive $ziparch zip archive instance
+     * @param string $archivepath file path to archive
+     * @param string $file path name of the file
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return bool success
+     */
+    private function archive_pathname($ziparch, $archivepath, $file,
+            file_progress $progress = null) {
+        // Record progress each time this function is called.
+        if ($progress) {
+            $progress->progress();
+        }
+
+        if (!file_exists($file)) {
+            return false;
+        }
+
+        if (is_file($file)) {
+            if (!is_readable($file)) {
+                return false;
+            }
+            return $ziparch->add_file_from_pathname($archivepath, $file);
+        }
+        if (is_dir($file)) {
+            if ($archivepath !== '') {
+                $ziparch->add_directory($archivepath);
+            }
+            $files = new DirectoryIterator($file);
+            foreach ($files as $file) {
+                if ($file->isDot()) {
+                    continue;
+                }
+                $newpath = $archivepath.'/'.$file->getFilename();
+                $this->archive_pathname($ziparch, $newpath, $file->getPathname(), $progress);
+            }
+            unset($files); // Release file handles.
+            return true;
+        }
+    }
+
+    /**
+     * Unzip file to given file path (real OS filesystem), existing files are overwritten.
+     *
+     * @todo MDL-31048 localise messages
+     * @param string|stored_file $archivefile full pathname of zip file or stored_file instance
+     * @param string $pathname target directory
+     * @param array $onlyfiles only extract files present in the array. The path to files MUST NOT
+     *              start with a /. Example: array('myfile.txt', 'directory/anotherfile.txt')
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @param bool $returnbool Whether to return a basic true/false indicating error state, or full per-file error
+     * details.
+     * @return bool|array list of processed files; false if error
+     */
+    public function extract_to_pathname($archivefile, $pathname,
+            array $onlyfiles = null, file_progress $progress = null, $returnbool = false) {
+        global $CFG;
+
+        if (!is_string($archivefile)) {
+            return $archivefile->extract_to_pathname($this, $pathname, $progress);
+        }
+
+        $processed = array();
+        $success = true;
+
+        $pathname = rtrim($pathname, '/');
+        if (!is_readable($archivefile)) {
+            return false;
+        }
+        $ziparch = new zip_archive();
+        if (!$ziparch->open($archivefile, file_archive::OPEN)) {
+            return false;
+        }
+
+        // Get the number of files (approx).
+        if ($progress) {
+            $approxmax = $ziparch->estimated_count();
+            $done = 0;
+        }
+
+        foreach ($ziparch as $info) {
+            // Notify progress.
+            if ($progress) {
+                $progress->progress($done, $approxmax);
+                $done++;
+            }
+
+            $size = $info->size;
+            $name = $info->pathname;
+            $origname = $name;
+
+            // File names cannot end with dots on Windows and trailing dots are replaced with underscore.
+            if ($CFG->ostype === 'WINDOWS') {
+                $name = preg_replace('~([^/]+)\.(/|$)~', '\1_\2', $name);
+            }
+
+            if ($name === '' or array_key_exists($name, $processed)) {
+                // Probably filename collisions caused by filename cleaning/conversion.
+                continue;
+            } else if (is_array($onlyfiles) && !in_array($origname, $onlyfiles)) {
+                // Skipping files which are not in the list.
+                continue;
+            }
+
+            if ($info->is_directory) {
+                $newdir = "$pathname/$name";
+                // directory
+                if (is_file($newdir) and !unlink($newdir)) {
+                    $processed[$name] = 'Can not create directory, file already exists'; // TODO: localise
+                    $success = false;
+                    continue;
+                }
+                if (is_dir($newdir)) {
+                    //dir already there
+                    $processed[$name] = true;
+                } else {
+                    if (mkdir($newdir, $CFG->directorypermissions, true)) {
+                        $processed[$name] = true;
+                    } else {
+                        $processed[$name] = 'Can not create directory'; // TODO: localise
+                        $success = false;
+                    }
+                }
+                continue;
+            }
+
+            $parts = explode('/', trim($name, '/'));
+            $filename = array_pop($parts);
+            $newdir = rtrim($pathname.'/'.implode('/', $parts), '/');
+
+            if (!is_dir($newdir)) {
+                if (!mkdir($newdir, $CFG->directorypermissions, true)) {
+                    $processed[$name] = 'Can not create directory'; // TODO: localise
+                    $success = false;
+                    continue;
+                }
+            }
+
+            $newfile = "$newdir/$filename";
+
+            if (strpos($newfile, './') > 1 || $name !== $origname) {
+                // The path to the entry contains a directory ending with dot. We cannot use extract_to() due to
+                // upstream PHP bugs #69477, #74619 and #77214. Extract the file from its stream which is slower but
+                // should work even in this case.
+                if (!$fp = fopen($newfile, 'wb')) {
+                    $processed[$name] = 'Can not write target file'; // TODO: localise.
+                    $success = false;
+                    continue;
+                }
+
+                if (!$fz = $ziparch->get_stream($info->index)) {
+                    $processed[$name] = 'Can not read file from zip archive'; // TODO: localise.
+                    $success = false;
+                    fclose($fp);
+                    continue;
+                }
+
+                while (!feof($fz)) {
+                    $content = fread($fz, 262143);
+                    fwrite($fp, $content);
+                }
+
+                fclose($fz);
+                fclose($fp);
+
+            } else {
+                if (!$fz = $ziparch->extract_to($pathname, $info->index)) {
+                    $processed[$name] = 'Can not read file from zip archive'; // TODO: localise.
+                    $success = false;
+                    continue;
+                }
+            }
+
+            // Check that the file was correctly created in the destination.
+            if (!file_exists($newfile)) {
+                $processed[$name] = 'Unknown error during zip extraction (file not created).'; // TODO: localise.
+                $success = false;
+                continue;
+            }
+
+            // Check that the size of extracted file matches the expectation.
+            if (filesize($newfile) !== $size) {
+                $processed[$name] = 'Unknown error during zip extraction (file size mismatch).'; // TODO: localise.
+                $success = false;
+                @unlink($newfile);
+                continue;
+            }
+
+            $processed[$name] = true;
+        }
+
+        $ziparch->close();
+
+        if ($returnbool) {
+            return $success;
+        } else {
+            return $processed;
+        }
+    }
+
+    /**
+     * Unzip file to given file path (real OS filesystem), existing files are overwritten.
+     *
+     * @todo MDL-31048 localise messages
+     * @param string|stored_file $archivefile full pathname of zip file or stored_file instance
+     * @param int $contextid context ID
+     * @param string $component component
+     * @param string $filearea file area
+     * @param int $itemid item ID
+     * @param string $pathbase file path
+     * @param int $userid user ID
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return array|bool list of processed files; false if error
+     */
+    public function extract_to_storage($archivefile, $contextid,
+            $component, $filearea, $itemid, $pathbase, $userid = NULL,
+            file_progress $progress = null) {
+        global $CFG;
+
+        if (!is_string($archivefile)) {
+            return $archivefile->extract_to_storage($this, $contextid, $component,
+                    $filearea, $itemid, $pathbase, $userid, $progress);
+        }
+
+        check_dir_exists($CFG->tempdir.'/zip');
+
+        $pathbase = trim($pathbase, '/');
+        $pathbase = ($pathbase === '') ? '/' : '/'.$pathbase.'/';
+        $fs = get_file_storage();
+
+        $processed = array();
+
+        $ziparch = new zip_archive();
+        if (!$ziparch->open($archivefile, file_archive::OPEN)) {
+            return false;
+        }
+
+        // Get the number of files (approx).
+        if ($progress) {
+            $approxmax = $ziparch->estimated_count();
+            $done = 0;
+        }
+
+        foreach ($ziparch as $info) {
+            // Notify progress.
+            if ($progress) {
+                $progress->progress($done, $approxmax);
+                $done++;
+            }
+
+            $size = $info->size;
+            $name = $info->pathname;
+
+            if ($name === '' or array_key_exists($name, $processed)) {
+                //probably filename collisions caused by filename cleaning/conversion
+                continue;
+            }
+
+            if ($info->is_directory) {
+                $newfilepath = $pathbase.$name.'/';
+                $fs->create_directory($contextid, $component, $filearea, $itemid, $newfilepath, $userid);
+                $processed[$name] = true;
+                continue;
+            }
+
+            $parts = explode('/', trim($name, '/'));
+            $filename = array_pop($parts);
+            $filepath = $pathbase;
+            if ($parts) {
+                $filepath .= implode('/', $parts).'/';
+            }
+
+            if ($size < 2097151) {
+                // Small file.
+                if (!$fz = $ziparch->get_stream($info->index)) {
+                    $processed[$name] = 'Can not read file from zip archive'; // TODO: localise
+                    continue;
+                }
+                $content = '';
+                $realfilesize = 0;
+                while (!feof($fz)) {
+                    $content .= fread($fz, 262143);
+                    $realfilesize = strlen($content); // Current file size.
+
+                    // More was read than was expected, which indicates a malformed/malicious archive.
+                    // Break and let the error handling below take care of the file clean up.
+                    if ($realfilesize > $size) {
+                        break;
+                    }
+                }
+                fclose($fz);
+                if (strlen($content) !== $size) {
+                    $processed[$name] = 'Unknown error during zip extraction'; // TODO: localise
+                    // something went wrong :-(
+                    unset($content);
+                    continue;
+                }
+
+                if ($file = $fs->get_file($contextid, $component, $filearea, $itemid, $filepath, $filename)) {
+                    if (!$file->delete()) {
+                        $processed[$name] = 'Can not delete existing file'; // TODO: localise
+                        continue;
+                    }
+                }
+                $file_record = new stdClass();
+                $file_record->contextid = $contextid;
+                $file_record->component = $component;
+                $file_record->filearea  = $filearea;
+                $file_record->itemid    = $itemid;
+                $file_record->filepath  = $filepath;
+                $file_record->filename  = $filename;
+                $file_record->userid    = $userid;
+                if ($fs->create_file_from_string($file_record, $content)) {
+                    $processed[$name] = true;
+                } else {
+                    $processed[$name] = 'Unknown error during zip extraction'; // TODO: localise
+                }
+                unset($content);
+                continue;
+
+            } else {
+                // large file, would not fit into memory :-(
+                $tmpfile = tempnam($CFG->tempdir.'/zip', 'unzip');
+                if (!$fp = fopen($tmpfile, 'wb')) {
+                    @unlink($tmpfile);
+                    $processed[$name] = 'Can not write temp file'; // TODO: localise
+                    continue;
+                }
+                if (!$fz = $ziparch->get_stream($info->index)) {
+                    @unlink($tmpfile);
+                    $processed[$name] = 'Can not read file from zip archive'; // TODO: localise
+                    continue;
+                }
+                $realfilesize = 0;
+                while (!feof($fz)) {
+                    $content = fread($fz, 262143);
+                    $numofbytes = fwrite($fp, $content);
+                    $realfilesize += $numofbytes; // Current file size.
+
+                    // More was read than was expected, which indicates a malformed/malicious archive.
+                    // Break and let the error handling below take care of the file clean up.
+                    if ($realfilesize > $size) {
+                        break;
+                    }
+                }
+                fclose($fz);
+                fclose($fp);
+                if (filesize($tmpfile) !== $size) {
+                    $processed[$name] = 'Unknown error during zip extraction'; // TODO: localise
+                    // something went wrong :-(
+                    @unlink($tmpfile);
+                    continue;
+                }
+
+                if ($file = $fs->get_file($contextid, $component, $filearea, $itemid, $filepath, $filename)) {
+                    if (!$file->delete()) {
+                        @unlink($tmpfile);
+                        $processed[$name] = 'Can not delete existing file'; // TODO: localise
+                        continue;
+                    }
+                }
+                $file_record = new stdClass();
+                $file_record->contextid = $contextid;
+                $file_record->component = $component;
+                $file_record->filearea  = $filearea;
+                $file_record->itemid    = $itemid;
+                $file_record->filepath  = $filepath;
+                $file_record->filename  = $filename;
+                $file_record->userid    = $userid;
+                if ($fs->create_file_from_pathname($file_record, $tmpfile)) {
+                    $processed[$name] = true;
+                } else {
+                    $processed[$name] = 'Unknown error during zip extraction'; // TODO: localise
+                }
+                @unlink($tmpfile);
+                continue;
+            }
+        }
+        $ziparch->close();
+        return $processed;
+    }
+
+    /**
+     * Returns array of info about all files in archive.
+     *
+     * @param string|file_archive $archivefile
+     * @return array of file infos
+     */
+    public function list_files($archivefile) {
+        if (!is_string($archivefile)) {
+            return $archivefile->list_files();
+        }
+
+        $ziparch = new zip_archive();
+        if (!$ziparch->open($archivefile, file_archive::OPEN)) {
+            return false;
+        }
+        $list = $ziparch->list_files();
+        $ziparch->close();
+        return $list;
+    }
+
+}