diff --git a/features/export.feature b/features/export.feature index c1d21a3e..ebdde6d9 100644 --- a/features/export.feature +++ b/features/export.feature @@ -1365,3 +1365,137 @@ Feature: Export content. """ """ + + Scenario: Export attaches terms, meta and comments to the right posts across batches + Given a WP install + And I run `wp site empty --yes` + And I run `wp post generate --count=150` + And I run `wp post create --post_title='Last post' --post_status=publish --tags_input='Zoo,apple,beta' --porcelain` + And save STDOUT as {LAST_POST_ID} + And I run `wp term create category Zoo --slug=zoo-category` + And I run `wp post term add {LAST_POST_ID} category zoo-category` + And I run `wp post meta add {LAST_POST_ID} my_meta_key my_meta_value` + And I run `wp post meta add {LAST_POST_ID} _edit_lock 123:1` + And I run `wp comment create --comment_post_ID={LAST_POST_ID} --comment_content='Approved comment' --comment_approved=1 --porcelain` + And save STDOUT as {COMMENT_ID} + And I run `wp comment meta add {COMMENT_ID} my_comment_meta_key my_comment_meta_value` + And I run `wp comment create --comment_post_ID={LAST_POST_ID} --comment_content='Spam comment' --comment_approved=spam` + And a wp-content/mu-plugins/skip-meta.php file: + """ + post_id !== get_the_ID(); + }, + 10, + 3 + ); + """ + And a check-export.php file: + """ + channel->item as $item ) { + $wp = $item->children( 'http://wordpress.org/export/1.2/' ); + $tags = array(); + $terms = array(); + foreach ( $item->category as $category ) { + if ( 'post_tag' === (string) $category['domain'] ) { + $tags[] = (string) $category['nicename']; + } + $terms[] = (string) $category; + } + $meta = array(); + foreach ( $wp->postmeta as $postmeta ) { + if ( in_array( (string) $postmeta->meta_key, array( '_pingme', '_encloseme' ), true ) ) { + continue; + } + $meta[] = (string) $postmeta->meta_key . '=' . (string) $postmeta->meta_value; + } + $comments = array(); + foreach ( $wp->comment as $comment ) { + $comment_meta = array(); + foreach ( $comment->commentmeta as $commentmeta ) { + $comment_meta[] = (string) $commentmeta->meta_key . '=' . (string) $commentmeta->meta_value; + } + $comments[] = (string) $comment->comment_content . ' [' . implode( ',', $comment_meta ) . ']'; + } + if ( (string) $wp->post_id === '{LAST_POST_ID}' ) { + ++$found; + // The terms should be in the same order as the database returns them for the post. Compare + // their names, as the database can return terms with the same name in any order. + $expected = wp_list_pluck( wp_get_object_terms( (int) $wp->post_id, get_object_taxonomies( 'post' ) ), 'name' ); + echo 'terms in database order: ' . ( $terms === $expected ? 'yes' : 'no' ) . "\n"; + sort( $tags ); + echo 'tags: ' . implode( ',', $tags ) . "\n"; + echo 'meta: ' . implode( ',', $meta ) . "\n"; + echo 'comments: ' . implode( ',', $comments ) . "\n"; + } elseif ( $tags || $meta || $comments || array( 'Uncategorized' ) !== $terms ) { + echo 'unexpected data on post ' . $wp->post_id . "\n"; + } + } + echo "found: $found\n"; + """ + + When I run `wp export --post_type=post` + Then save STDOUT 'Writing to file %s' as {EXPORT_FILE} + + When I run `wp eval-file check-export.php {EXPORT_FILE}` + Then STDOUT should be: + """ + terms in database order: yes + tags: apple,beta,zoo + meta: my_meta_key=my_meta_value + comments: Approved comment [my_comment_meta_key=my_comment_meta_value] + found: 1 + """ + + # Duplicate IDs in --post__in leave gaps in the keys of the list of post IDs. + When I run `wp post list --post_type=post --orderby=ID --order=ASC --format=ids` + Then save STDOUT as {POST_IDS} + + When I run `wp eval 'echo implode( ",", array_merge( ...array_map( function ( $id ) { return array( $id, $id ); }, explode( " ", "{POST_IDS}" ) ) ) );'` + Then save STDOUT as {DUPLICATED_POST_IDS} + + Given a wp-content/mu-plugins/count-batches.php file: + """ + , meta: array<\stdClass>, comments: array<\stdClass>}> + */ + private $batch_data = []; + + /** + * Positions of the post IDs in $post_ids, to find the batch a post belongs to. + * + * @var array|null + */ + private $post_id_positions; + public function __construct( $filters = [] ) { $this->filters = wp_parse_args( $filters, self::$defaults ); @@ -156,9 +170,21 @@ public function exportify_post( $post ) { // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound -- Calling native WordPress hook. $post->post_excerpt = apply_filters( 'the_excerpt_export', $post->post_excerpt ); $post->is_sticky = is_sticky( $post->ID ) ? 1 : 0; - $post->terms = self::get_terms_for_post( $post ); - $post->meta = self::get_meta_for_post( $post ); - $post->comments = $this->get_comments_for_post( $post ); + + if ( ! isset( $this->batch_data[ $post->ID ] ) ) { + $this->load_batch_data( $post->ID ); + } + + if ( isset( $this->batch_data[ $post->ID ] ) ) { + $post->terms = $this->batch_data[ $post->ID ]['terms']; + $post->meta = self::filter_meta_for_export( $this->batch_data[ $post->ID ]['meta'] ); + $post->comments = $this->batch_data[ $post->ID ]['comments']; + unset( $this->batch_data[ $post->ID ] ); + } else { + $post->terms = self::get_terms_for_post( $post ); + $post->meta = self::get_meta_for_post( $post ); + $post->comments = $this->get_comments_for_post( $post ); + } // phpcs:ignore WordPress.WP.GlobalVariablesOverride.Prohibited -- Revert back to original. $GLOBALS['post'] = $previous_global_post; return $post; @@ -413,6 +439,138 @@ private function process_orphaned_terms( $terms ) { return $terms; } + /** + * Load the terms, meta and comments of the batch of posts starting with the given one. + * + * Querying them for a whole batch at once avoids several queries per exported post. + * The data is the same as what get_terms_for_post(), get_meta_for_post() and + * get_comments_for_post() return for each post, except that the meta is not filtered yet. + * + * @param int $post_id ID of the first post of the batch. + */ + private function load_batch_data( $post_id ) { + global $wpdb; + + if ( null === $this->post_id_positions ) { + $this->post_id_positions = array_flip( array_map( 'intval', array_values( $this->post_ids ) ) ); + } + + $this->batch_data = []; + if ( ! isset( $this->post_id_positions[ $post_id ] ) ) { + return; + } + + // Load the same chunk of posts that the posts iterator queries, which returns them in no particular order. + $start = intdiv( $this->post_id_positions[ $post_id ], self::QUERY_CHUNK ) * self::QUERY_CHUNK; + $post_ids = array_map( 'intval', array_slice( $this->post_ids, $start, self::QUERY_CHUNK ) ); + if ( ! $post_ids ) { + return; + } + $in_ids = implode( ',', $post_ids ); + + $terms_by_post = array_fill_keys( $post_ids, [] ); + $meta_by_post = array_fill_keys( $post_ids, [] ); + $comments_by_post = array_fill_keys( $post_ids, [] ); + + // Terms, queried per post type as the taxonomies depend on it. + // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- integers only + $post_types = $wpdb->get_results( "SELECT ID, post_type FROM $wpdb->posts WHERE ID IN ($in_ids)" ); + $ids_by_type = []; + foreach ( $post_types as $row ) { + $ids_by_type[ $row->post_type ][] = (int) $row->ID; + } + foreach ( $ids_by_type as $post_type => $ids ) { + $taxonomies = get_object_taxonomies( $post_type ); + if ( empty( $taxonomies ) ) { + continue; + } + $terms = wp_get_object_terms( $ids, $taxonomies, [ 'fields' => 'all_with_object_id' ] ); + if ( is_wp_error( $terms ) ) { + continue; + } + foreach ( $terms as $term ) { + if ( ! $term instanceof \WP_Term ) { + continue; + } + // Set by wp_get_object_terms() for 'all_with_object_id'. + $object_id = (int) $term->object_id; // @phpstan-ignore property.notFound + if ( isset( $terms_by_post[ $object_id ] ) ) { + $terms_by_post[ $object_id ][] = $term; + } + } + } + + // The terms are ordered by name in the database's collation. Order terms with the same name, + // e.g. a category and a tag, by their term_taxonomy_id like the query for a single post returns them. + foreach ( $terms_by_post as &$terms ) { + $terms = self::order_terms_with_same_name( $terms ); + } + unset( $terms ); + + // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- integers only + $meta_from_db = $wpdb->get_results( "SELECT * FROM $wpdb->postmeta WHERE post_id IN ($in_ids) ORDER BY post_id, meta_id" ); + // The meta is filtered in exportify_post(), while the post it belongs to is the global post. + foreach ( $meta_from_db as $meta ) { + $meta_by_post[ (int) $meta->post_id ][] = $meta; + } + + if ( ! isset( $this->filters['skip_comments'] ) ) { + // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- integers only + $comments = $wpdb->get_results( "SELECT * FROM $wpdb->comments WHERE comment_post_ID IN ($in_ids) AND comment_approved <> 'spam' ORDER BY comment_post_ID, comment_ID" ); + if ( ! empty( $comments ) ) { + $comment_meta = []; + $comment_ids = implode( ',', array_map( 'intval', wp_list_pluck( $comments, 'comment_ID' ) ) ); + // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- integers only + foreach ( $wpdb->get_results( "SELECT * FROM $wpdb->commentmeta WHERE comment_id IN ($comment_ids) ORDER BY comment_id, meta_id" ) as $meta ) { + $comment_meta[ (int) $meta->comment_id ][] = $meta; + } + + foreach ( $comments as $comment ) { + $comment->meta = isset( $comment_meta[ (int) $comment->comment_ID ] ) ? $comment_meta[ (int) $comment->comment_ID ] : []; + $comments_by_post[ (int) $comment->comment_post_ID ][] = $comment; + } + } + } + + foreach ( $post_ids as $id ) { + $this->batch_data[ $id ] = [ + 'terms' => $terms_by_post[ $id ], + 'meta' => $meta_by_post[ $id ], + 'comments' => $comments_by_post[ $id ], + ]; + } + } + + /** + * Order runs of terms with the same name by their term_taxonomy_id, keeping the order of the runs. + * + * @param array<\WP_Term> $terms Terms ordered by name. + * @return array<\WP_Term> + */ + private static function order_terms_with_same_name( $terms ) { + $ordered = []; + $count = count( $terms ); + $i = 0; + while ( $i < $count ) { + $run = [ $terms[ $i ] ]; + ++$i; + while ( $i < $count && $terms[ $i ]->name === $run[0]->name ) { + $run[] = $terms[ $i ]; + ++$i; + } + if ( count( $run ) > 1 ) { + usort( + $run, + static function ( $a, $b ) { + return $a->term_taxonomy_id <=> $b->term_taxonomy_id; + } + ); + } + array_push( $ordered, ...$run ); + } + return $ordered; + } + private static function get_terms_for_post( $post ) { $taxonomies = get_object_taxonomies( $post->post_type ); if ( empty( $taxonomies ) ) { @@ -425,6 +583,20 @@ private static function get_meta_for_post( $post ) { global $wpdb; $meta_for_export = []; $meta_from_db = $wpdb->get_results( $wpdb->prepare( "SELECT * FROM $wpdb->postmeta WHERE post_id = %d", $post->ID ) ); + return self::filter_meta_for_export( $meta_from_db ); + } + + /** + * Remove the meta that should not be exported. + * + * Called while the post the meta belongs to is the global post, as callbacks + * of the `wxr_export_skip_postmeta` filter may rely on it. + * + * @param array<\stdClass> $meta_from_db Meta rows of a post. + * @return array<\stdClass> + */ + private static function filter_meta_for_export( $meta_from_db ) { + $meta_for_export = []; foreach ( $meta_from_db as $meta ) { if ( '_edit_lock' === $meta->meta_key ) { continue; diff --git a/src/WP_Export_WXR_Formatter.php b/src/WP_Export_WXR_Formatter.php index 870eeb74..0a6779d8 100644 --- a/src/WP_Export_WXR_Formatter.php +++ b/src/WP_Export_WXR_Formatter.php @@ -288,7 +288,10 @@ protected function terms( $terms ) { protected function comment_meta( $comment ) { global $wpdb; - $metas = $wpdb->get_results( $wpdb->prepare( "SELECT * FROM $wpdb->commentmeta WHERE comment_id = %d", $comment->comment_ID ) ); + // The export query already loads the comment meta, see WP_Export_Query::exportify_post(). + $metas = isset( $comment->meta ) && is_array( $comment->meta ) + ? $comment->meta + : $wpdb->get_results( $wpdb->prepare( "SELECT * FROM $wpdb->commentmeta WHERE comment_id = %d", $comment->comment_ID ) ); if ( ! $metas ) { return new Oxymel(); }