compare local files with the branch (git blob SHA-1s), list changes and deletions * upload -> create blobs for binary files, then build the new tree in chunks * (text files are sent inline inside the tree requests) * finalize -> one commit for the whole change set + fast-forward the branch * * Only changed files are sent, so an auto-push for one post is a small commit. */ if ( ! defined( 'ABSPATH' ) ) { exit; } class CFSP_Deploy_GitHub { const STATE = 'gh-deploy'; const MAX_FILE_SIZE = 26214400; // 25 MiB: the Cloudflare Pages per-file limit also applies to Git builds. const TREE_BYTES = 2097152; // Inline text per tree request. const TREE_ENTRIES = 300; // Files per tree request. Halved automatically when GitHub times out. const TREE_MIN = 25; const BLOB_PAUSE_US = 750000; // Stay under GitHub's ~80 content-creating requests per minute. const TEXT_EXT = array( 'html', 'htm', 'css', 'js', 'mjs', 'json', 'map', 'xml', 'xsl', 'txt', 'svg', 'csv', 'webmanifest', '' ); /** @return array|WP_Error */ public static function prepare( $job ) { $gh = new CFSP_GitHub(); $head = $gh->get_head(); if ( is_wp_error( $head ) && 'cfsp_empty_repo' === $head->get_error_code() ) { $init = $gh->init_repo(); if ( is_wp_error( $init ) ) { return $init; } CFSP_Logger::log( 'Initialized empty repository with a README.' ); $head = $gh->get_head(); } if ( is_wp_error( $head ) ) { return $head; } // What's currently in the branch. $remote = array(); if ( $head ) { $tree = $gh->get_tree( $head['tree'] ); if ( is_wp_error( $tree ) ) { return $tree; } $remote = $tree['files']; if ( $tree['truncated'] ) { $pushed = CFSP_Deployer::read_state( 'gh-pushed' ); $remote = $pushed['files'] ?? array(); CFSP_Logger::error( 'Repository tree is too large to list; comparing with the last pushed state instead.' ); } } $prefix = CFSP_Settings::github_prefix(); $root = trailingslashit( CFSP_Settings::export_dir() ); $cache = CFSP_Deployer::read_state( 'gh-hash-cache' ); $new_cache = array(); $local = array(); $blobs = array(); $entries = array(); foreach ( CFSP_Deployer::files() as $rel => $file ) { $size = $file->getSize(); if ( $size > self::MAX_FILE_SIZE ) { CFSP_Logger::error( "Skipped {$rel}: larger than 25 MiB (Cloudflare Pages limit)." ); continue; } $mtime = $file->getMTime(); if ( isset( $cache[ $rel ] ) && $cache[ $rel ][0] === $size && $cache[ $rel ][1] === $mtime ) { $sha = $cache[ $rel ][2]; } else { $sha = self::git_sha( file_get_contents( $root . $rel ) ); } $new_cache[ $rel ] = array( $size, $mtime, $sha ); $path = $prefix . $rel; $local[ $path ] = $sha; if ( ( $remote[ $path ] ?? null ) === $sha ) { continue; } if ( self::is_text( $rel ) ) { $entries[] = array( 'path' => $path, 'rel' => $rel ); } else { $blobs[] = array( 'path' => $path, 'rel' => $rel ); } } if ( ! $local ) { return new WP_Error( 'cfsp_empty', 'Nothing to push: the export directory is empty. Run a full export first.' ); } // Cloudflare Pages reads _headers and _redirects from the output directory. $generated = array( '_headers' => CFSP_Deployer::headers_file(), '_redirects' => CFSP_Deployer::redirects_file(), ); foreach ( $generated as $name => $content ) { if ( '' === $content ) { continue; } $path = $prefix . $name; $sha = self::git_sha( $content ); $local[ $path ] = $sha; if ( ( $remote[ $path ] ?? null ) !== $sha ) { $entries[] = array( 'path' => $path, 'content' => $content ); } } // Files in the repo that no longer exist in the export. // Media moved to R2 is an expected, intentional deletion and doesn't count toward the safety limit. $media_prefix = null; if ( CFSP_Media::exclude_uploads() ) { $media_prefix = $prefix . ltrim( CFSP_Media::uploads_rel_path(), '/' ); } $deletes = 0; $media_deletes = 0; foreach ( $remote as $path => $sha ) { if ( isset( $local[ $path ] ) || ( $prefix && 0 !== strpos( $path, $prefix ) ) ) { continue; } if ( self::is_protected( substr( $path, strlen( $prefix ) ) ) ) { continue; } $entries[] = array( 'path' => $path, 'delete' => true ); if ( $media_prefix && 0 === strpos( $path, $media_prefix ) ) { $media_deletes++; } else { $deletes++; } } if ( $media_deletes ) { CFSP_Logger::log( sprintf( 'Removing %d media file(s) from the repository. They are served from R2 now.', $media_deletes ) ); } // Guard against wiping the repository after a broken export. $limit = max( 50, (int) ( count( $remote ) * 0.5 ) ); if ( $deletes > $limit && ! apply_filters( 'cfsp_github_allow_mass_delete', false, $deletes ) ) { return new WP_Error( 'cfsp_api', sprintf( 'Refusing to delete %d files from the repository in one push. Check the export, or allow it with the cfsp_github_allow_mass_delete filter.', $deletes ) ); } CFSP_Deployer::write_state( 'gh-hash-cache', $new_cache ); $changes = count( $blobs ) + count( $entries ); CFSP_Deployer::write_state( self::STATE, array( 'head' => $head ? $head['commit'] : null, 'base_tree' => $head ? $head['tree'] : null, 'tree' => null, 'message' => self::commit_message( $job, $changes - $deletes - $media_deletes, $deletes + $media_deletes ), 'blobs' => $blobs, 'entries' => $entries, 'local' => $local, 'nochange' => 0 === $changes, ) ); return array( 'files' => count( $local ), 'missing' => $changes, ); } /** @return int|WP_Error Items still to send. */ public static function upload( $deadline ) { $s = CFSP_Deployer::read_state( self::STATE ); $gh = new CFSP_GitHub(); $root = trailingslashit( CFSP_Settings::export_dir() ); if ( ! $s ) { return new WP_Error( 'cfsp_restart', 'Deploy state missing.' ); } // 1) Binary files need their own blob. $made = 0; while ( $s['blobs'] && time() < $deadline ) { $item = $s['blobs'][0]; $content = is_file( $root . $item['rel'] ) ? file_get_contents( $root . $item['rel'] ) : false; if ( false !== $content ) { $sha = $gh->create_blob( $content ); if ( is_wp_error( $sha ) ) { CFSP_Deployer::write_state( self::STATE, $s ); return $sha; } $s['entries'][] = array( 'path' => $item['path'], 'sha' => $sha ); $made++; } array_shift( $s['blobs'] ); if ( 0 === $made % 10 ) { CFSP_Deployer::write_state( self::STATE, $s ); } usleep( self::BLOB_PAUSE_US ); } if ( $made ) { CFSP_Logger::log( sprintf( 'Uploaded %d binary file(s) to GitHub.', $made ) ); } CFSP_Deployer::write_state( self::STATE, $s ); // 2) Build the new tree in chunks, each on top of the previous one. while ( ! $s['blobs'] && $s['entries'] && time() < $deadline ) { $chunk = array(); $bytes = 0; $taken = 0; $base = $s['tree'] ? $s['tree'] : $s['base_tree']; $max = (int) ( $s['chunk'] ?? self::TREE_ENTRIES ); $limit = (int) ( self::TREE_BYTES * $max / self::TREE_ENTRIES ); foreach ( $s['entries'] as $e ) { if ( count( $chunk ) >= $max ) { break; } $entry = array( 'path' => $e['path'], 'mode' => '100644', 'type' => 'blob', ); if ( ! empty( $e['delete'] ) ) { if ( $base ) { $entry['sha'] = null; $chunk[] = $entry; } } elseif ( isset( $e['sha'] ) ) { $entry['sha'] = $e['sha']; $chunk[] = $entry; } else { $content = $e['content'] ?? ( is_file( $root . $e['rel'] ) ? file_get_contents( $root . $e['rel'] ) : false ); if ( false !== $content ) { if ( $chunk && $bytes + strlen( $content ) > $limit ) { break; } if ( self::is_utf8( $content ) ) { $entry['content'] = $content; } else { $sha = $gh->create_blob( $content ); if ( is_wp_error( $sha ) ) { return $sha; } $entry['sha'] = $sha; } $bytes += strlen( $content ); $chunk[] = $entry; } } $taken++; } if ( $chunk ) { $tree = $gh->create_tree( $base, $chunk ); if ( is_wp_error( $tree ) ) { // GitHub gives up on large trees in big repositories ("Sorry, your request timed out"). // Send smaller batches instead of repeating the same request. if ( $max > self::TREE_MIN && false !== stripos( $tree->get_error_message(), 'timed out' ) ) { $s['chunk'] = max( self::TREE_MIN, (int) floor( $max / 2 ) ); CFSP_Deployer::write_state( self::STATE, $s ); CFSP_Logger::log( sprintf( 'GitHub timed out building %d file change(s); retrying in batches of %d.', count( $chunk ), $s['chunk'] ) ); continue; } return $tree; // State untouched: this chunk is retried. } $s['tree'] = $tree; CFSP_Logger::log( sprintf( 'Sent %d file change(s) to GitHub (%s).', count( $chunk ), size_format( $bytes ) ) ); } $s['entries'] = array_slice( $s['entries'], $taken ); CFSP_Deployer::write_state( self::STATE, $s ); } return count( $s['blobs'] ) + count( $s['entries'] ); } /** @return array|WP_Error */ public static function finalize() { $s = CFSP_Deployer::read_state( self::STATE ); if ( ! $s ) { return new WP_Error( 'cfsp_restart', 'Deploy state missing.' ); } $repo_url = 'https://github.com/' . CFSP_Settings::get( 'gh_repo' ); if ( ! empty( $s['nochange'] ) || ! $s['tree'] || $s['tree'] === $s['base_tree'] ) { CFSP_Deployer::write_state( self::STATE, array() ); return array( 'url' => $repo_url, 'id' => $s['head'], 'nochange' => true, ); } $gh = new CFSP_GitHub(); $commit = $gh->create_commit( $s['message'], $s['tree'], $s['head'] ? array( $s['head'] ) : array() ); if ( is_wp_error( $commit ) ) { return $commit; } $ref = $s['head'] ? $gh->update_ref( $commit['sha'] ) : $gh->create_ref( $commit['sha'] ); if ( is_wp_error( $ref ) ) { $status = $ref->get_error_data()['status'] ?? 0; if ( in_array( $status, array( 409, 422 ), true ) ) { // Someone else pushed to the branch meanwhile: redo the diff on top of the new tip. return new WP_Error( 'cfsp_restart', 'The branch changed during the push; retrying on top of the latest commit.' ); } return $ref; } CFSP_Deployer::write_state( 'gh-pushed', array( 'commit' => $commit['sha'], 'files' => $s['local'], ) ); CFSP_Deployer::write_state( self::STATE, array() ); return array( 'url' => $commit['html_url'] ?? $repo_url . '/commit/' . $commit['sha'], 'id' => $commit['sha'], ); } /* ------------------------------------------------------------------ */ public static function git_sha( $content ) { return sha1( 'blob ' . strlen( $content ) . "\0" . $content ); } private static function is_text( $rel ) { return in_array( strtolower( pathinfo( $rel, PATHINFO_EXTENSION ) ), self::TEXT_EXT, true ); } private static function is_utf8( $content ) { if ( false !== strpos( $content, "\0" ) ) { return false; } return function_exists( 'mb_check_encoding' ) ? mb_check_encoding( $content, 'UTF-8' ) : (bool) preg_match( '//u', $content ); } /** Repository files the plugin never deletes (README, CI config, dotfiles, custom domain, ...). */ private static function is_protected( $rel ) { foreach ( explode( '/', $rel ) as $segment ) { if ( '' !== $segment && '.' === $segment[0] ) { return true; } } $protected = preg_match( '#^(readme|license|licence|cname|wrangler\.(toml|json|jsonc)|package\.json)(\.[a-z]+)?$#i', $rel ); return (bool) apply_filters( 'cfsp_github_protected', $protected, $rel ); } private static function commit_message( $job, $changed, $deleted ) { $template = CFSP_Settings::get( 'gh_commit_message' ); $reason = ! empty( $job['reason'] ) ? $job['reason'] : 'Update'; $count = $changed . ' changed' . ( $deleted ? ", {$deleted} deleted" : '' ); $message = strtr( $template ? $template : '{reason} ({count})', array( '{reason}' => $reason, '{count}' => $count ) ); return mb_substr( $message, 0, 500 ); } }