Fix handling of --output-format json flag

- Don't treat it as deprecated on stable and beta channels. Before, it
  would give confusing and incorrect output:

  ```
  warning: the 'output-format' flag is considered deprecated
    |
    = warning: see issue #44136 <https://github.com/rust-lang/rust/issues/44136> for more information

  error: json output format isn't supported for doc generation
  ```
  Both of those are wrong: output-format isn't deprecated, and json
  output is supported.

- Require -Z unstable-options for `--output-format json`

  Previously, it was allowed by default on nightly, which made it hard
  to realize the flag wouldn't be accepted on beta or stable.
  Note that this still allows `--output-format html`, which has been
  stable since 1.0.

- Remove unnecessary double-checking of the feature gate when parsing
  the output format
- Add custom run-make test since compiletest passes -Zunstable-options
    by default

src/librustdoc/config.rs

@@ -379,6 +379,17 @@ fn println_condition(condition: Condition) {
             }
         }
 
+        // check for `--output-format=json`
+        if !matches!(matches.opt_str("output-format").as_deref(), None | Some("html"))
+            && !matches.opt_present("show-coverage")
+            && !nightly_options::is_unstable_enabled(matches)
+        {
+            rustc_session::early_error(
+                error_format,
+                "the -Z unstable-options flag must be passed to enable --output-format for documentation generation (see https://github.com/rust-lang/rust/issues/76578)",
+            );
+        }
+
         let to_check = matches.opt_strs("check-theme");
         if !to_check.is_empty() {
             let paths = theme::load_css_paths(static_files::themes::LIGHT.as_bytes());
@@ -575,13 +586,7 @@ fn println_condition(condition: Condition) {
         let output_format = match matches.opt_str("output-format") {
             Some(s) => match OutputFormat::try_from(s.as_str()) {
                 Ok(out_fmt) => {
-                    if out_fmt.is_json()
+                    if !out_fmt.is_json() && show_coverage {
-                        && !(show_coverage || nightly_options::match_is_nightly_build(matches))
-                    {
-                        diag.struct_err("json output format isn't supported for doc generation")
-                            .emit();
-                        return Err(1);
-                    } else if !out_fmt.is_json() && show_coverage {
                         diag.struct_err(
                             "html output format isn't supported for the --show-coverage option",
                         )
@@ -703,16 +708,10 @@ fn println_condition(condition: Condition) {
 
 /// Prints deprecation warnings for deprecated options
 fn check_deprecated_options(matches: &getopts::Matches, diag: &rustc_errors::Handler) {
-    let deprecated_flags = ["input-format", "output-format", "no-defaults", "passes"];
+    let deprecated_flags = ["input-format", "no-defaults", "passes"];
 
     for flag in deprecated_flags.iter() {
         if matches.opt_present(flag) {
-            if *flag == "output-format"
-                && (matches.opt_present("show-coverage")
-                    || nightly_options::match_is_nightly_build(matches))
-            {
-                continue;
-            }
             let mut err = diag.struct_warn(&format!("the `{}` flag is deprecated", flag));
             err.note(
                 "see issue #44136 <https://github.com/rust-lang/rust/issues/44136> \

src/test/run-make/unstable-flag-required/Makefile

@@ -0,0 +1,4 @@
+-include ../../run-make-fulldeps/tools.mk
+
+all:
+	$(RUSTDOC) --output-format=json x.html 2>&1 | diff - output-format-json.stderr

src/test/run-make/unstable-flag-required/README.md

@@ -0,0 +1,3 @@
+This is a collection of tests that verify `--unstable-options` is required.
+It should eventually be removed in favor of UI tests once compiletest stops
+passing --unstable-options by default (#82639).

src/test/run-make/unstable-flag-required/output-format-json.stderr

@@ -0,0 +1,2 @@
+error: the -Z unstable-options flag must be passed to enable --output-format for documentation generation (see https://github.com/rust-lang/rust/issues/76578)
+

src/test/run-make/unstable-flag-required/x.rs

@@ -0,0 +1 @@
+// nothing to see here

src/test/rustdoc-ui/output-format-html-stable.rs

@@ -0,0 +1,4 @@
+// compile-flags: --output-format html
+// check-pass
+// This tests that `--output-format html` is accepted without `-Z unstable-options`,
+// since it has been stable since 1.0.

src/tools/compiletest/src/runtest.rs

@@ -1600,7 +1600,7 @@ fn document(&self, out_dir: &Path) -> ProcRes {
             .args(&self.props.compile_flags);
 
         if self.config.mode == RustdocJson {
-            rustdoc.arg("--output-format").arg("json");
+            rustdoc.arg("--output-format").arg("json").arg("-Zunstable-options");
         }
 
         if let Some(ref linker) = self.config.linker {