dump database -> write archive -> finish. * * @package Shiftbox */ // Reading the site's own tables in batches cannot use the object cache, so direct queries are expected here. // Every query goes through $wpdb->prepare() with %i (identifiers), %s and %d placeholders. // phpcs:disable WordPress.WP.AlternativeFunctions, WordPress.PHP.NoSilencedErrors.Discouraged, WordPress.DB.DirectDatabaseQuery defined('ABSPATH') || exit; /** * Exporter. */ class Shiftbox_Exporter { /** * Folders and files that are never worth backing up. * * @var string[] */ const ALWAYS_EXCLUDE = array( 'shiftbox-backups', 'ai1wm-backups', 'updraft', 'backups-dup-lite', 'backups-dup-pro', 'wpvividbackups', 'backupwordpress', 'backup-db', 'backups', 'cache', 'et-cache', 'litespeed', 'upgrade', 'upgrade-temp-backup', 'wflogs', 'debug.log', 'advanced-cache.php', 'object-cache.php', 'db.php', ); /** * Default options for a new export. * * @param array $opts Requested options. * @return array */ public static function normalize_options(array $opts) { $keys = array('no_media', 'no_themes', 'no_plugins', 'no_database', 'no_inactive_plugins', 'no_inactive_themes', 'no_revisions', 'no_spam'); $out = array(); foreach ($keys as $key) { $out[$key] = ! empty($opts[$key]); } $out['tag'] = isset($opts['tag']) ? sanitize_key($opts['tag']) : ''; return $out; } /** * Initial state for a new export job. * * @param array $opts Options. * @return array */ public static function initial_state(array $opts) { $opts = self::normalize_options($opts); return array( 'opts' => $opts, 'step' => 'scan', 'message' => __('Finding files…', 'shiftbox'), 'excludes' => self::build_excludes($opts), 'queue' => array(''), 'files' => 0, 'bytes' => 0, 'list_len' => 0, 'name' => Shiftbox_Storage::new_backup_name($opts['tag']), ); } /** * Builds the exclusion list (paths relative to wp-content). * * @param array $opts Options. * @return string[] */ private static function build_excludes(array $opts) { $ex = self::ALWAYS_EXCLUDE; $own = dirname(plugin_basename(SHIFTBOX_FILE)); $ex[] = 'plugins/' . $own; $storage_rel = self::relative(Shiftbox_Storage::dir()); if (null !== $storage_rel) { $ex[] = $storage_rel; } if ($opts['no_media']) { $up = wp_upload_dir(null, false); $rel = self::relative($up['basedir']); if (null !== $rel) { $ex[] = $rel; } } if ($opts['no_themes']) { $ex[] = 'themes'; } if ($opts['no_plugins']) { $ex[] = 'plugins'; } if ($opts['no_inactive_plugins'] && ! $opts['no_plugins']) { if (! function_exists('get_plugins')) { require_once ABSPATH . 'wp-admin/includes/plugin.php'; } $active = (array) get_option('active_plugins', array()); foreach (array_keys(get_plugins()) as $file) { if (! in_array($file, $active, true)) { $dir = dirname($file); $ex[] = 'plugins/' . ('.' === $dir ? $file : $dir); } } } if ($opts['no_inactive_themes'] && ! $opts['no_themes']) { $keep = array(get_template(), get_stylesheet()); foreach ((array) glob(get_theme_root() . '/*', GLOB_ONLYDIR) as $dir) { if (! in_array(basename($dir), $keep, true)) { $rel = self::relative($dir); if (null !== $rel) { $ex[] = $rel; } } } } $custom = (string) Shiftbox_Settings::get('custom_exclude'); foreach (preg_split('/\R/', $custom) as $line) { $line = trim($line, "/ \t"); if ('' !== $line) { $ex[] = $line; } } /** * Filters paths (relative to wp-content) left out of backups. * * @param string[] $ex Paths. * @param array $opts Export options. */ return array_values(array_unique(apply_filters('shiftbox_export_excludes', $ex, $opts))); } /** * Path relative to the content folder, or null when outside it. * * @param string $path Absolute path. * @return string|null */ private static function relative($path) { $base = Shiftbox_Storage::content_dir() . '/'; $path = wp_normalize_path(untrailingslashit($path)); if (0 === strpos($path . '/', $base) && strlen($path) >= strlen($base)) { return substr($path, strlen($base)); } return null; } /** * Runs work until the deadline. * * @param Shiftbox_Job $job Job. * @param float $deadline microtime(true) limit. * @return void * @throws RuntimeException On failure. */ public static function tick(Shiftbox_Job $job, $deadline) { $d = &$job->data; while (microtime(true) < $deadline) { switch ($d['step']) { case 'scan': self::scan($job, $deadline); break; case 'database': self::database($job, $deadline); break; case 'archive': self::archive($job, $deadline); break; case 'finish': self::finish($job); return; default: return; } } } /** * Breadth-first file scan, resumable via a queue of folders. * * @param Shiftbox_Job $job Job. * @param float $deadline Deadline. * @return void * @throws RuntimeException On failure. */ private static function scan(Shiftbox_Job $job, $deadline) { $d = &$job->data; $base = Shiftbox_Storage::content_dir(); $list = fopen($job->path('files.txt'), 'c+b'); ftruncate($list, (int) $d['list_len']); fseek($list, 0, SEEK_END); $limit = (int) Shiftbox_Settings::get('max_backup'); while (! empty($d['queue']) && microtime(true) < $deadline) { $rel_dir = array_shift($d['queue']); $abs_dir = '' === $rel_dir ? $base : $base . '/' . $rel_dir; $entries = @scandir($abs_dir); if (false === $entries) { $job->log('Skipped unreadable folder: ' . $rel_dir); continue; } foreach ($entries as $name) { if ('.' === $name || '..' === $name) { continue; } $rel = '' === $rel_dir ? $name : $rel_dir . '/' . $name; if (self::is_excluded($rel, $d['excludes']) || false !== strpos($rel, "\n")) { continue; } $abs = $abs_dir . '/' . $name; if (is_dir($abs)) { if (! is_link($abs)) { $d['queue'][] = $rel; } } elseif (is_file($abs) && is_readable($abs)) { if (strlen('content/' . $rel) > 65000) { continue; } fwrite($list, $rel . "\n"); ++$d['files']; $d['bytes'] += (int) @filesize($abs); } } if ($limit > 0 && $d['bytes'] > $limit) { fclose($list); /* translators: 1: backup size limit, 2: size found so far. */ throw new RuntimeException(esc_html(sprintf(__('This backup would be larger than your limit of %1$s (already %2$s of files). Raise the limit in Settings or leave out media, plugins or themes.', 'shiftbox'), Shiftbox_Settings::human($limit), size_format($d['bytes'])))); } } fflush($list); $d['list_len'] = ftell($list); fclose($list); /* translators: 1: file count, 2: total size. */ $d['message'] = sprintf(__('Found %1$s files (%2$s)…', 'shiftbox'), number_format_i18n($d['files']), size_format($d['bytes'])); $d['progress'] = 5; if (empty($d['queue'])) { $d['step'] = $d['opts']['no_database'] ? 'archive' : 'database'; if ('database' === $d['step']) { self::init_database($job); } else { self::init_archive($job); } } } /** * Whether a relative path matches an exclusion. * * @param string $rel Relative path. * @param string[] $excludes Exclusions. * @return bool */ private static function is_excluded($rel, array $excludes) { foreach ($excludes as $ex) { if ($rel === $ex || 0 === strpos($rel, $ex . '/')) { return true; } } return false; } /** * Prepares the database dump. * * @param Shiftbox_Job $job Job. * @return void */ private static function init_database(Shiftbox_Job $job) { global $wpdb; $tables = array(); foreach ((array) $wpdb->get_results('SHOW FULL TABLES', ARRAY_N) as $row) { if (isset($row[1]) && 'BASE TABLE' === $row[1] && 0 === strpos($row[0], $wpdb->prefix)) { $tables[] = $row[0]; } } /** * Filters the tables included in a backup. * * @param string[] $tables Table names. */ $tables = array_values(apply_filters('shiftbox_export_tables', $tables)); $job->data['db'] = array( 'tables' => $tables, 'index' => 0, 'table' => null, 'sql' => 0, 'rows' => 0, ); file_put_contents($job->path('database.sql'), ''); $job->data['message'] = __('Saving the database…', 'shiftbox'); } /** * Dumps tables in batches. * * @param Shiftbox_Job $job Job. * @param float $deadline Deadline. * @return void * @throws RuntimeException On failure. */ private static function database(Shiftbox_Job $job, $deadline) { global $wpdb; $d = &$job->data; $db = &$d['db']; $rows = (int) Shiftbox_Settings::profile()['rows']; $fh = fopen($job->path('database.sql'), 'c+b'); ftruncate($fh, (int) $db['sql']); fseek($fh, 0, SEEK_END); if (0 === (int) $db['sql']) { fwrite($fh, '-- Shiftbox ' . SHIFTBOX_VERSION . " database export\n"); } while (microtime(true) < $deadline && $db['index'] < count($db['tables'])) { $table = $db['tables'][$db['index']]; $name = '{{SBPFX}}' . self::dump_identifier(substr($table, strlen($wpdb->prefix))); if (null === $db['table']) { $create = $wpdb->get_row($wpdb->prepare('SHOW CREATE TABLE %i', $table), ARRAY_N); if (empty($create[1])) { $job->log('Skipped table without definition: ' . $table); ++$db['index']; continue; } $sql = preg_replace('/`' . preg_quote($wpdb->prefix, '/') . '/', '`{{SBPFX}}', $create[1]); $sql = str_replace(array("\r\n", "\n", "\r"), ' ', $sql); fwrite($fh, 'DROP TABLE IF EXISTS `' . $name . "`;\n" . $sql . ";\n"); $cols = array(); $binary = array(); $columns = (array) $wpdb->get_results($wpdb->prepare('SHOW COLUMNS FROM %i', $table), ARRAY_A); foreach ($columns as $col) { $cols[] = $col['Field']; $binary[] = (bool) preg_match('/blob|binary/i', $col['Type']); } $pk = null; $keys = (array) $wpdb->get_results($wpdb->prepare('SHOW KEYS FROM %i WHERE Key_name = %s', $table, 'PRIMARY'), ARRAY_A); if (1 === count($keys)) { foreach ($columns as $col) { if ($col['Field'] === $keys[0]['Column_name'] && preg_match('/int/i', $col['Type'])) { $pk = $col['Field']; } } } $db['table'] = array( 'cols' => $cols, 'binary' => $binary, 'pk' => $pk, 'last' => null, 'offset' => 0, 'where' => self::table_filter($table, $d['opts']), ); } $t = &$db['table']; $colsql = '`' . implode('`,`', array_map(array(__CLASS__, 'dump_identifier'), $t['cols'])) . '`'; $result = self::read_batch($table, $t, $rows); if ($wpdb->last_error) { fclose($fh); /* translators: 1: table, 2: database error. */ throw new RuntimeException(esc_html(sprintf(__('Could not read table %1$s: %2$s', 'shiftbox'), $table, $wpdb->last_error))); } $buffer = ''; $pk_idx = $t['pk'] ? array_search($t['pk'], $t['cols'], true) : false; foreach ((array) $result as $row) { $vals = array(); foreach ($row as $i => $v) { if (null === $v) { $vals[] = 'NULL'; } elseif ($t['binary'][$i]) { $vals[] = '' === $v ? "''" : '0x' . bin2hex($v); } else { // Escapes the value for the backup's SQL file (written to disk, not run here). // The placeholder escape esc_sql() adds for "%" is removed so values stay identical on any site. $vals[] = "'" . str_replace(array("\n", "\r"), array('\n', '\r'), $wpdb->remove_placeholder_escape(esc_sql($v))) . "'"; } } $tuple = '(' . implode(',', $vals) . ')'; if ('' === $buffer) { $buffer = 'INSERT INTO `' . $name . '` (' . $colsql . ') VALUES ' . $tuple; } else { $buffer .= ',' . $tuple; } if (strlen($buffer) > 524288) { fwrite($fh, $buffer . ";\n"); $buffer = ''; } if (false !== $pk_idx) { $t['last'] = $row[$pk_idx]; } ++$db['rows']; } if ('' !== $buffer) { fwrite($fh, $buffer . ";\n"); } $t['offset'] += count((array) $result); if (count((array) $result) < $rows) { unset($t); $db['table'] = null; ++$db['index']; } else { unset($t); } /* translators: 1: table number, 2: table count. */ $d['message'] = sprintf(__('Saving the database (table %1$d of %2$d)…', 'shiftbox'), min($db['index'] + 1, count($db['tables'])), count($db['tables'])); $d['progress'] = 5 + (int) (20 * ($db['index'] / max(1, count($db['tables'])))); } fflush($fh); $db['sql'] = ftell($fh); fclose($fh); if ($db['index'] >= count($db['tables'])) { $d['step'] = 'archive'; self::init_archive($job); } } /** * Extra conditions per table (revisions, spam, transients), as * column / NOT LIKE / pattern triples that are bound with placeholders. * * @param string $table Table. * @param array $opts Options. * @return array[] */ private static function table_filter($table, array $opts) { global $wpdb; $where = array(); if ($table === $wpdb->options) { $where[] = array('option_name', 'NOT LIKE', $wpdb->esc_like('_transient_') . '%'); $where[] = array('option_name', 'NOT LIKE', $wpdb->esc_like('_site_transient_') . '%'); } if ($opts['no_revisions'] && $table === $wpdb->posts) { $where[] = array('post_type', 'NOT LIKE', $wpdb->esc_like('revision')); } if ($opts['no_spam'] && $table === $wpdb->comments) { $where[] = array('comment_approved', 'NOT LIKE', $wpdb->esc_like('spam')); } return $where; } /** * Reads the next batch of rows from a table. * * Table and column names are bound with %i, values with %s / %d, so nothing * from the database is ever placed into the query text directly. The only * text added to the query is a fixed list of placeholders. * * @param string $table Table. * @param array $t Table state (cols, pk, last, offset, where). * @param int $rows Batch size. * @return array */ private static function read_batch($table, array $t, $rows) { global $wpdb; $args = $t['cols']; $args[] = $table; $count = 0; foreach ((array) $t['where'] as $w) { if (is_array($w) && 3 === count($w) && 'NOT LIKE' === $w[1]) { $args[] = (string) $w[0]; $args[] = (string) $w[2]; ++$count; } } $select = implode(', ', array_fill(0, count($t['cols']), '%i')); $filter = implode('', array_fill(0, $count, ' AND %i NOT LIKE %s')); if (! $t['pk']) { $args[] = (int) $t['offset']; $args[] = (int) $rows; // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- $select and $filter contain only placeholders. return (array) $wpdb->get_results($wpdb->prepare("SELECT {$select} FROM %i WHERE 1=1{$filter} LIMIT %d, %d", $args), ARRAY_N); } if (null === $t['last']) { $args[] = $t['pk']; $args[] = (int) $rows; // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- $select and $filter contain only placeholders. return (array) $wpdb->get_results($wpdb->prepare("SELECT {$select} FROM %i WHERE 1=1{$filter} ORDER BY %i ASC LIMIT %d", $args), ARRAY_N); } $args[] = $t['pk']; $args[] = (int) $t['last']; $args[] = $t['pk']; $args[] = (int) $rows; // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- $select and $filter contain only placeholders. return (array) $wpdb->get_results($wpdb->prepare("SELECT {$select} FROM %i WHERE 1=1{$filter} AND %i > %d ORDER BY %i ASC LIMIT %d", $args), ARRAY_N); } /** * Quotes a column name for the INSERT statements written into the backup's * SQL file (a file, not a query run here): backticks are doubled. * * @param string $name Identifier. * @return string */ public static function dump_identifier($name) { return str_replace('`', '``', (string) $name); } /** * Prepares archive writing. * * @param Shiftbox_Job $job Job. * @return void * @throws RuntimeException When disk space is short. */ private static function init_archive(Shiftbox_Job $job) { $d = &$job->data; $sql = isset($d['db']['sql']) ? (int) $d['db']['sql'] : 0; $needed = ($d['bytes'] + $sql) * 1.02 + 10485760; $free = Shiftbox_Storage::free_space(); if ($free >= 0 && $free < $needed) { /* translators: 1: needed space, 2: free space. */ throw new RuntimeException(esc_html(sprintf(__('Not enough disk space: the backup needs about %1$s but only %2$s is free. Delete old backups or leave out media.', 'shiftbox'), size_format($needed), size_format($free)))); } $limit = (int) Shiftbox_Settings::get('max_backup'); if ($limit > 0 && ($d['bytes'] + $sql) > $limit) { /* translators: 1: limit, 2: size. */ throw new RuntimeException(esc_html(sprintf(__('This backup would be %2$s, above your limit of %1$s. Raise the limit in Settings or leave out some content.', 'shiftbox'), Shiftbox_Settings::human($limit), size_format($d['bytes'] + $sql)))); } file_put_contents($job->path('package.json'), wp_json_encode(self::manifest($job), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); $d['archive'] = array( 'part' => Shiftbox_Storage::dir() . '/' . $d['name'] . '.part', 'bytes' => 0, 'phase' => 0, 'offset' => 0, 'cur' => null, 'done' => 0, 'total' => $d['bytes'] + $sql, ); $d['message'] = __('Packing files…', 'shiftbox'); } /** * Information about this site stored inside the backup. * * @param Shiftbox_Job $job Job. * @return array */ private static function manifest(Shiftbox_Job $job) { global $wpdb, $wp_version; $up = wp_upload_dir(null, false); return array( 'generator' => 'Shiftbox', 'version' => SHIFTBOX_VERSION, 'format' => 1, 'created' => time(), 'home' => get_option('home'), 'siteurl' => get_option('siteurl'), 'abspath' => wp_normalize_path(untrailingslashit(ABSPATH)), 'content_dir' => Shiftbox_Storage::content_dir(), 'uploads_dir' => wp_normalize_path(untrailingslashit($up['basedir'])), 'uploads_url' => untrailingslashit($up['baseurl']), 'prefix' => $wpdb->prefix, 'wp' => $wp_version, 'php' => PHP_VERSION, 'db' => $wpdb->db_version(), 'multisite' => is_multisite(), 'template' => get_template(), 'stylesheet' => get_stylesheet(), 'plugins' => (array) get_option('active_plugins', array()), 'files' => (int) $job->data['files'], 'bytes' => (int) $job->data['bytes'], 'database' => empty($job->data['opts']['no_database']), 'options' => $job->data['opts'], ); } /** * Writes entries into the archive. * * @param Shiftbox_Job $job Job. * @param float $deadline Deadline. * @return void * @throws RuntimeException On failure. */ private static function archive(Shiftbox_Job $job, $deadline) { $d = &$job->data; $a = &$d['archive']; $chunk = (int) Shiftbox_Settings::profile()['chunk']; $base = Shiftbox_Storage::content_dir(); $out = fopen($a['part'], 'c+b'); if (! $out) { throw new RuntimeException(esc_html(__('Could not write the backup file inside uploads/shiftbox-backups.', 'shiftbox'))); } ftruncate($out, (int) $a['bytes']); fseek($out, 0, SEEK_END); if (0 === (int) $a['bytes']) { fwrite($out, Shiftbox_Archive::MAGIC); } $list = null; while (microtime(true) < $deadline) { if (null === $a['cur']) { if (0 === $a['phase']) { $a['cur'] = self::start_entry($out, $job->path('package.json'), 'package.json'); $a['phase'] = $d['opts']['no_database'] ? 2 : 1; } elseif (1 === $a['phase']) { $a['cur'] = self::start_entry($out, $job->path('database.sql'), 'database.sql'); $a['phase'] = 2; } else { if (! $list) { $list = fopen($job->path('files.txt'), 'rb'); fseek($list, (int) $a['offset']); } $line = fgets($list); if (false === $line) { $d['step'] = 'finish'; break; } $a['offset'] = ftell($list); $rel = rtrim($line, "\n"); $a['cur'] = self::start_entry($out, $base . '/' . $rel, 'content/' . $rel); if (null === $a['cur']) { $job->log('Skipped (missing or unreadable): ' . $rel); } } continue; } $cur = &$a['cur']; $in = @fopen($cur['src'], 'rb'); if ($in && $cur['done'] > 0) { fseek($in, (int) $cur['done']); } while ($cur['done'] < $cur['size'] && microtime(true) < $deadline) { $want = (int) min($chunk, $cur['size'] - $cur['done']); $data = $in ? fread($in, $want) : ''; if (false === $data || '' === $data) { $data = str_repeat("\0", $want); } $written = fwrite($out, $data); if (false === $written || $written !== strlen($data)) { fclose($out); throw new RuntimeException(esc_html(__('Writing the backup failed. The disk may be full.', 'shiftbox'))); } $cur['done'] += $written; $a['done'] += $written; } if ($in) { fclose($in); } if ($cur['done'] >= $cur['size']) { unset($cur); $a['cur'] = null; } else { unset($cur); } } if ($list) { fclose($list); } if ('finish' === $d['step']) { fwrite($out, Shiftbox_Archive::END); } fflush($out); $a['bytes'] = ftell($out); fclose($out); $d['progress'] = 25 + (int) (74 * ($a['done'] / max(1, $a['total']))); /* translators: 1: done size, 2: total size. */ $d['message'] = sprintf(__('Packing files: %1$s of %2$s', 'shiftbox'), size_format($a['done']), size_format($a['total'])); } /** * Human list of what a backup contains. * * @param array $opts Options. * @return string */ private static function included_labels(array $opts) { $parts = array(); if (empty($opts['no_database'])) { $parts[] = __('database', 'shiftbox'); } if (empty($opts['no_media'])) { $parts[] = __('media', 'shiftbox'); } if (empty($opts['no_themes'])) { $parts[] = empty($opts['no_inactive_themes']) ? __('all themes', 'shiftbox') : __('active theme', 'shiftbox'); } if (empty($opts['no_plugins'])) { $parts[] = empty($opts['no_inactive_plugins']) ? __('all plugins', 'shiftbox') : __('active plugins', 'shiftbox'); } return implode(', ', $parts); } /** * Writes an entry header and returns the in-progress entry, or null to skip. * * @param resource $out Archive handle. * @param string $src Source file. * @param string $path Path inside the archive. * @return array|null */ private static function start_entry($out, $src, $path) { if (! is_file($src) || ! is_readable($src)) { return null; } $size = (int) @filesize($src); fwrite($out, Shiftbox_Archive::entry_header($path, $size, (int) @filemtime($src))); return array( 'src' => $src, 'size' => $size, 'done' => 0, ); } /** * Final rename, retention and cleanup. * * @param Shiftbox_Job $job Job. * @return void */ private static function finish(Shiftbox_Job $job) { $d = &$job->data; $final = Shiftbox_Storage::dir() . '/' . $d['name']; if (! @rename($d['archive']['part'], $final)) { throw new RuntimeException(esc_html(__('Could not finalize the backup file.', 'shiftbox'))); } $d['step'] = 'done'; $d['progress'] = 100; $d['result'] = array( 'name' => $d['name'], 'size' => (float) filesize($final), 'download' => add_query_arg( array( 'action' => 'shiftbox_download', 'file' => rawurlencode($d['name']), '_wpnonce' => wp_create_nonce('shiftbox_download'), ), admin_url('admin-ajax.php') ), 'list' => admin_url('admin.php?page=shiftbox-backups'), 'files' => number_format_i18n((int) $d['files']), 'included' => self::included_labels($d['opts']), 'seconds' => max(1, time() - (int) $d['created']), ); /* translators: %s: backup size. */ $d['message'] = sprintf(__('Backup ready (%s).', 'shiftbox'), size_format($d['result']['size'])); if ('scheduled' === $d['opts']['tag']) { Shiftbox_Scheduler::apply_retention(); } /** * Fires when a backup file is complete. Add-ons use this to copy it to cloud storage. * * @param string $final Absolute path to the backup. * @param array $data Job state. */ do_action('shiftbox_backup_created', $final, $d); } }