Preserve documentation and hook string semantics

lib/class-command.php

@@ -152,7 +152,14 @@ protected function _get_phpdoc_data( $path, $format = 'json' ) {
 		$output = parse_files( $files, $path );
 
 		if ( 'json' == $format ) {
-			return json_encode( $output, JSON_PRETTY_PRINT );
+			$json = json_encode( $output, JSON_PRETTY_PRINT );
+
+			if ( false === $json ) {
+				WP_CLI::error( sprintf( 'Problem encoding the data from %1$s as JSON: %2$s', $path, json_last_error_msg() ) );
+				exit;
+			}
+
+			return $json;
 		}
 
 		return $output;

lib/class-hook-reflector.php

@@ -10,10 +10,16 @@
 class Hook_Reflector extends BaseReflector {
 
 	/**
+	 * Get the hook name as it is spelled in the source.
+	 *
+	 * The name is printed from the source expression instead of read from the
+	 * interpreted string value. Interpreting escape sequences may produce bytes
+	 * that are not valid UTF-8, which cannot be encoded as JSON.
+	 *
 	 * @return string
 	 */
 	public function getName() {
-		$printer = new \PhpParser\PrettyPrinter\Standard();
+		$printer = new Pretty_Printer();
 		return $this->cleanupName( $printer->prettyPrintExpr( $this->node->args[0]->value ) );
 	}
 
@@ -26,8 +32,10 @@ private function cleanupName( $name ) {
 		$matches = array();
 
 		// quotes on both ends of a string
-		if ( preg_match( '/^[\'"]([^\'"]*)[\'"]$/', $name, $matches ) ) {
-			return $matches[1];
+		// The quoted body may contain the other quote character, as in "it's",
+		// or an escaped copy of the quote that delimits it, as in 'it\'s'.
+		if ( preg_match( '/^([\'"])((?:(?!\1)[^\\\\]|\\\\.)*)\1$/s', $name, $matches ) ) {
+			return $matches[2];
 		}
 
 		// two concatenated things, last one of them a variable

lib/class-pretty-printer.php

@@ -5,7 +5,57 @@
 /**
  * Extends default printer for arguments.
  */
-class Pretty_Printer extends \PhpParser\PrettyPrinter\Standard {
+class Pretty_Printer extends \phpDocumentor\Reflection\PrettyPrinter {
+	/**
+	 * Print names as they appeared before PHP-Parser's name resolution.
+	 *
+	 * PHP-Parser represents resolved global names as fully-qualified names. The
+	 * leading namespace separator is useful in an AST, but adding it to exported
+	 * source expressions changes the established JSON output.
+	 *
+	 * Single-segment fully-qualified names are therefore printed without the
+	 * leading backslash. Inside a namespaced file the printed form denotes a
+	 * namespaced symbol rather than the global one, for example `\Foo::BAR` is
+	 * printed as `Foo::BAR`, which in a namespaced file would resolve to
+	 * `Vendor\Foo::BAR`. This is an accepted limitation because the parser
+	 * targets global-namespace WordPress core code.
+	 *
+	 * @param \PhpParser\Node\Name\FullyQualified $node Fully-qualified name.
+	 *
+	 * @return string Printed name.
+	 */
+	protected function pName_FullyQualified( \PhpParser\Node\Name\FullyQualified $node ): string {
+		$name = $node->toString();
+
+		return false === strpos( $name, '\\' ) ? $name : '\\' . $name;
+	}
+
+	/**
+	 * Print heredoc and nowdoc strings with their delimiters.
+	 *
+	 * The parent printer returns PHP-Parser's `rawValue` attribute so that
+	 * escape sequences are not interpreted. For heredoc and nowdoc strings that
+	 * attribute holds the body only, without the `<<<LABEL` delimiters, which is
+	 * no longer a PHP expression. Those are printed by the default printer,
+	 * which does not interpret escape sequences in doc strings either.
+	 *
+	 * @param \PhpParser\Node\Scalar\String_ $node String.
+	 *
+	 * @return string Printed string.
+	 */
+	public function pScalar_String( \PhpParser\Node\Scalar\String_ $node ): string {
+		$kind = $node->getAttribute( 'kind' );
+
+		if (
+			\PhpParser\Node\Scalar\String_::KIND_HEREDOC === $kind ||
+			\PhpParser\Node\Scalar\String_::KIND_NOWDOC === $kind
+		) {
+			return \PhpParser\PrettyPrinter\Standard::pScalar_String( $node );
+		}
+
+		return parent::pScalar_String( $node );
+	}
+
 	/**
 	 * Pretty prints an argument.
 	 *

lib/runner.php

@@ -90,7 +90,7 @@ function parse_files( $files, $root ) {
 				$out['constants'][] = array(
 					'name'  => $constant->getShortName(),
 					'line'  => $constant->getLineNumber(),
-					'value' => $constant->getValue(),
+					'value' => export_expression( $constant->getNode()->value ),
 				);
 			}
 
@@ -102,7 +102,7 @@ function parse_files( $files, $root ) {
 				$func = array(
 					'name'      => $function->getShortName(),
 					'namespace' => $function->getNamespace(),
-					'aliases'   => $function->getNamespaceAliases(),
+					'aliases'   => strip_global_namespace_prefixes( $function->getNamespaceAliases() ),
 					'line'      => $function->getLineNumber(),
 					'end_line'  => $function->getNode()->getAttribute( 'endLine' ),
 					'arguments' => export_arguments( $function->getArguments() ),
@@ -132,8 +132,8 @@ function parse_files( $files, $root ) {
 					'end_line'   => $class->getNode()->getAttribute( 'endLine' ),
 					'final'      => $class->isFinal(),
 					'abstract'   => $class->isAbstract(),
-					'extends'    => $class->getParentClass(),
-					'implements' => $class->getInterfaces(),
+					'extends'    => strip_global_namespace_prefix( $class->getParentClass() ),
+					'implements' => strip_global_namespace_prefixes( $class->getInterfaces() ),
 					'properties' => export_properties( $class->getProperties(), $class_setup_blueprints, $path ),
 					'methods'    => export_methods( $class->getMethods(), $class_setup_blueprints, $path ),
 					'doc'        => $class_doc,
@@ -149,33 +149,101 @@ function parse_files( $files, $root ) {
 		throw $e;
 	}
 
-	/*
-	 * nikic/php-parser in version 3 started adding a namespace prefix
-	 * at the start of global names, but this is different than how the
-	 * documentation was previously generated. this removes those prefixes
-	 * by removing a leading reverse solidus (\) when no other reverse
-	 * solidus appears before the end of a sequence of PHP identifier
-	 * characters.
-	 */
-	array_walk_recursive(
-		$output,
-		static function( &$value ) {
-			if ( is_string( $value ) ) {
-				// "\wp_kses()" -> "wp_kses()"
-				$without_global_namespace = preg_replace(
-					'~(^|\p{Z})\\\\([A-Z_a-z\x80-\xFF][0-9A-Z_a-z\x80-\xFF]*)([:(\p{Z}]|->|$)~',
-					'$1$2$3',
-					$value,
-				);
+	return $output;
+}
 
-				if ( $value !== $without_global_namespace ) {
-					$value = $without_global_namespace;
-				}
-			}
-		}
+/**
+ * Remove a synthetic leading namespace prefix from a global name.
+ *
+ * @param mixed $name Name to normalize.
+ *
+ * @return mixed
+ */
+function strip_global_namespace_prefix( $name ) {
+	if ( ! is_string( $name ) ) {
+		return $name;
+	}
+
+	return preg_replace(
+		'~^\\\\([A-Z_a-z\x80-\xFF][0-9A-Z_a-z\x80-\xFF]*)([:(\p{Z}]|->|$)~',
+		'$1$2',
+		$name
 	);
+}
 
-	return $output;
+/**
+ * Remove synthetic leading namespace prefixes from global names.
+ *
+ * @param array $names Names to normalize.
+ *
+ * @return array
+ */
+function strip_global_namespace_prefixes( array $names ) {
+	foreach ( $names as $key => $name ) {
+		$names[ $key ] = strip_global_namespace_prefix( $name );
+	}
+
+	return $names;
+}
+
+/**
+ * Export an expression without PHP-Parser's synthetic global namespace prefixes.
+ *
+ * @param null|\PhpParser\Node\Expr $expression Expression to export.
+ *
+ * @return null|string
+ */
+function export_expression( $expression ) {
+	if ( null === $expression ) {
+		return null;
+	}
+
+	static $printer = null;
+
+	if ( null === $printer ) {
+		$printer = new Pretty_Printer();
+	}
+
+	return $printer->prettyPrintExpr( $expression );
+}
+
+/**
+ * Remove synthetic global namespace prefixes from inline DocBlock references.
+ *
+ * A special exception is made for text appearing in `<code>` and `<pre>` tags, as code
+ * samples are reproduced verbatim and any prefix appearing in them was written by hand.
+ *
+ * @param string $text Formatted DocBlock text.
+ *
+ * @return string
+ */
+function strip_global_namespace_prefixes_from_inline_references( $text ) {
+	// Non-naturally occurring string to use as temporary replacement.
+	$replacement_string = '{{{{{}}}}}';
+
+	// Replace inline tag openings within 'code' and 'pre' tags with replacement string.
+	$text = preg_replace_callback(
+		"/(<code[^>]*>)(.+)(?=<\/code>)/sU",
+		function ( $matches ) use ( $replacement_string ) {
+			return str_replace( '{@', $replacement_string, $matches[1] . $matches[2] );
+		},
+		$text
+	);
+
+	$text = preg_replace_callback(
+		'~{@(?:link|see)\s+([^}\s]+)~',
+		static function( $matches ) {
+			return str_replace(
+				$matches[1],
+				strip_global_namespace_prefix( $matches[1] ),
+				$matches[0]
+			);
+		},
+		$text
+	);
+
+	// Restore inline tag openings into code blocks.
+	return str_replace( $replacement_string, '{@', $text );
 }
 
 /**
@@ -506,8 +574,12 @@ function export_docblock( $element, array $inherited_setup_blueprints = array(),
 	}
 
 	$output = array(
-		'description'      => preg_replace( '/[\n\r]+/', ' ', $short_description ),
-		'long_description' => format_long_description( strip_docblock_code_snippet_fences( $raw_long_description, $fences ) ),
+		'description'      => strip_global_namespace_prefixes_from_inline_references(
+			preg_replace( '/[\n\r]+/', ' ', $short_description )
+		),
+		'long_description' => strip_global_namespace_prefixes_from_inline_references(
+			format_long_description( strip_docblock_code_snippet_fences( $raw_long_description, $fences ) )
+		),
 		'tags'             => array(),
 	);
 
@@ -527,19 +599,21 @@ function export_docblock( $element, array $inherited_setup_blueprints = array(),
 
 		$tag_data = array(
 			'name'    => $tag->getName(),
-			'content' => preg_replace( '/[\n\r]+/', ' ', format_description( $tag->getDescription() ) ),
+			'content' => strip_global_namespace_prefixes_from_inline_references(
+				preg_replace( '/[\n\r]+/', ' ', format_description( $tag->getDescription() ) )
+			),
 		);
 		if ( method_exists( $tag, 'getTypes' ) ) {
-			$tag_data['types'] = $tag->getTypes();
+			$tag_data['types'] = strip_global_namespace_prefixes( $tag->getTypes() );
 		}
 		if ( method_exists( $tag, 'getLink' ) ) {
-			$tag_data['link'] = $tag->getLink();
+			$tag_data['link'] = strip_global_namespace_prefix( $tag->getLink() );
 		}
 		if ( method_exists( $tag, 'getVariableName' ) ) {
 			$tag_data['variable'] = $tag->getVariableName();
 		}
 		if ( method_exists( $tag, 'getReference' ) ) {
-			$tag_data['refers'] = $tag->getReference();
+			$tag_data['refers'] = strip_global_namespace_prefix( $tag->getReference() );
 		}
 		if ( method_exists( $tag, 'getVersion' ) ) {
 			// Version string.
@@ -549,7 +623,9 @@ function export_docblock( $element, array $inherited_setup_blueprints = array(),
 			}
 			// Description string.
 			if ( method_exists( $tag, 'getDescription' ) ) {
-				$description = preg_replace( '/[\n\r]+/', ' ', format_description( $tag->getDescription() ) );
+				$description = strip_global_namespace_prefixes_from_inline_references(
+					preg_replace( '/[\n\r]+/', ' ', format_description( $tag->getDescription() ) )
+				);
 				if ( ! empty( $description ) ) {
 					$tag_data['description'] = $description;
 				}
@@ -697,8 +773,8 @@ function export_arguments( array $arguments ) {
 	foreach ( $arguments as $argument ) {
 		$output[] = array(
 			'name'    => $argument->getName(),
-			'default' => $argument->getDefault(),
-			'type'    => $argument->getType(),
+			'default' => export_expression( $argument->getNode()->default ),
+			'type'    => strip_global_namespace_prefix( $argument->getType() ),
 		);
 	}
 
@@ -720,7 +796,7 @@ function export_properties( array $properties, array $inherited_setup_blueprints
 			'name'        => $property->getName(),
 			'line'        => $property->getLineNumber(),
 			'end_line'    => $property->getNode()->getAttribute( 'endLine' ),
-			'default'     => $property->getDefault(),
+			'default'     => export_expression( $property->getNode()->default ),
 //			'final' => $property->isFinal(),
 			'static'      => $property->isStatic(),
 			'visibility'  => $property->getVisibility(),
@@ -746,7 +822,7 @@ function export_methods( array $methods, array $inherited_setup_blueprints = arr
 		$method_data = array(
 			'name'       => $method->getShortName(),
 			'namespace'  => $method->getNamespace(),
-			'aliases'    => $method->getNamespaceAliases(),
+			'aliases'    => strip_global_namespace_prefixes( $method->getNamespaceAliases() ),
 			'line'       => $method->getLineNumber(),
 			'end_line'   => $method->getNode()->getAttribute( 'endLine' ),
 			'final'      => $method->isFinal(),
@@ -1283,7 +1359,7 @@ function export_uses( array $uses ) {
 				case 'methods':
 					$out[ $type ][] = array(
 						'name'     => $name[1],
-						'class'    => $name[0],
+						'class'    => strip_global_namespace_prefix( $name[0] ),
 						'static'   => $element->isStatic(),
 						'line'     => $element->getLineNumber(),
 						'end_line' => $element->getNode()->getAttribute( 'endLine' ),
@@ -1292,6 +1368,7 @@ function export_uses( array $uses ) {
 
 				default:
 				case 'functions':
+					$name = strip_global_namespace_prefix( $name );
 					$out[ $type ][] = array(
 						'name'     => $name,
 						'line'     => $element->getLineNumber(),

tests/phpunit/tests/export/docblocks.inc

@@ -40,6 +40,21 @@ function test_func( $var, $num ) {
 	return true;
 }
 
+/**
+ * Tests special characters in documentation.
+ *
+ * ```php
+ * true === wp_is_valid_utf8( '✏' );
+ * false === wp_is_valid_utf8( "just \xC0 test" );
+ * ```
+ */
+function test_special_characters() {}
+
+/**
+ * \xC0 starts this description.
+ */
+function test_leading_escape_sequence() {}
+
 /**
  * This is a class docblock.
  *