Skip to content

Commit 514f3aa

Browse files
sypharsyphar
committed
Add rustdoc JSON download endpoints & about-page
1 parent 107722e commit 514f3aa

File tree

7 files changed

+330
-3
lines changed

7 files changed

+330
-3
lines changed

‎src/storage/mod.rs‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,16 @@ use fn_error_context::context;
2222
use futures_util::stream::BoxStream;
2323
use mime::Mime;
2424
use path_slash::PathExt;
25-
use std::iter;
25+
use serde_with::{DeserializeFromStr, SerializeDisplay};
2626
use std::{
2727
fmt, fs,
2828
io::{self, BufReader},
29+
num::ParseIntError,
2930
ops::RangeInclusive,
3031
path::{Path, PathBuf},
3132
sync::Arc,
3233
};
34+
use std::{iter, str::FromStr};
3335
use tokio::{io::AsyncWriteExt, runtime::Runtime};
3436
use tracing::{error, info_span, instrument, trace};
3537
use walkdir::WalkDir;
@@ -815,14 +817,25 @@ pub(crate) fn rustdoc_archive_path(name: &str, version: &str) -> String {
815817
format!("rustdoc/{name}/{version}.zip")
816818
}
817819

818-
#[derive(strum::Display, Debug, PartialEq, Eq)]
820+
#[derive(strum::Display, Debug, PartialEq, Eq, Clone, SerializeDisplay, DeserializeFromStr)]
819821
#[strum(serialize_all = "snake_case")]
820822
pub(crate) enum RustdocJsonFormatVersion {
821823
#[strum(serialize = "{0}")]
822824
Version(u16),
823825
Latest,
824826
}
825827

828+
impl FromStr for RustdocJsonFormatVersion {
829+
type Err = ParseIntError;
830+
fn from_str(s: &str) -> Result<Self, Self::Err> {
831+
if s == "latest" {
832+
Ok(RustdocJsonFormatVersion::Latest)
833+
} else {
834+
s.parse::<u16>().map(RustdocJsonFormatVersion::Version)
835+
}
836+
}
837+
}
838+
826839
pub(crate) fn rustdoc_json_path(
827840
name: &str,
828841
version: &str,
@@ -842,6 +855,25 @@ pub(crate) fn source_archive_path(name: &str, version: &str) -> String {
842855
mod test {
843856
use super::*;
844857
use std::env;
858+
use test_case::test_case;
859+
860+
#[test_case("latest", RustdocJsonFormatVersion::Latest)]
861+
#[test_case("42", RustdocJsonFormatVersion::Version(42))]
862+
fn test_json_format_version(input: &str, expected: RustdocJsonFormatVersion) {
863+
// test Display
864+
assert_eq!(expected.to_string(), input);
865+
// test FromStr
866+
assert_eq!(expected, input.parse().unwrap());
867+
868+
let json_input = format!("\"{input}\"");
869+
// test Serialize
870+
assert_eq!(serde_json::to_string(&expected).unwrap(), json_input);
871+
// test Deserialize
872+
assert_eq!(
873+
serde_json::from_str::<RustdocJsonFormatVersion>(&json_input).unwrap(),
874+
expected
875+
);
876+
}
845877

846878
#[test]
847879
fn test_get_file_list() -> Result<()> {

‎src/web/error.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ pub enum AxumNope {
4444
OwnerNotFound,
4545
#[error("Requested crate does not have specified version")]
4646
VersionNotFound,
47+
#[error("Requested release doesn't have docs for the given target")]
48+
TargetNotFound,
4749
#[error("Search yielded no results")]
4850
NoResults,
4951
#[error("Unauthorized: {0}")]
@@ -77,6 +79,14 @@ impl AxumNope {
7779
message: "no such build".into(),
7880
status: StatusCode::NOT_FOUND,
7981
},
82+
AxumNope::TargetNotFound => {
83+
// user tried to navigate to a target that doesn't exist
84+
ErrorInfo {
85+
title: "The requested target does not exist",
86+
message: "no such target".into(),
87+
status: StatusCode::NOT_FOUND,
88+
}
89+
}
8090
AxumNope::CrateNotFound => {
8191
// user tried to navigate to a crate that doesn't exist
8292
// TODO: Display the attempted crate and a link to a search for said crate

‎src/web/routes.rs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,10 +301,26 @@ pub(super) fn build_axum_routes() -> AxumRouter {
301301
"/crate/{name}/{version}/download",
302302
get_internal(super::rustdoc::download_handler),
303303
)
304+
.route_with_tsr(
305+
"/crate/{name}/{version}/json",
306+
get_internal(super::rustdoc::json_download_handler),
307+
)
308+
.route_with_tsr(
309+
"/crate/{name}/{version}/json/{format_version}",
310+
get_internal(super::rustdoc::json_download_handler),
311+
)
304312
.route(
305313
"/crate/{name}/{version}/target-redirect/{*path}",
306314
get_internal(super::rustdoc::target_redirect_handler),
307315
)
316+
.route_with_tsr(
317+
"/crate/{name}/{version}/{target}/json",
318+
get_internal(super::rustdoc::json_download_handler),
319+
)
320+
.route_with_tsr(
321+
"/crate/{name}/{version}/{target}/json/{format_version}",
322+
get_internal(super::rustdoc::json_download_handler),
323+
)
308324
.route(
309325
"/{name}/badge.svg",
310326
get_internal(super::rustdoc::badge_handler),

‎src/web/rustdoc.rs‎

Lines changed: 196 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
use crate::{
44
AsyncStorage, Config, InstanceMetrics, RUSTDOC_STATIC_STORAGE_PREFIX,
55
db::Pool,
6-
storage::rustdoc_archive_path,
6+
storage::{RustdocJsonFormatVersion, rustdoc_archive_path, rustdoc_json_path},
77
utils,
88
web::{
99
MetaData, ReqVersion, axum_cached_redirect, axum_parse_uri_with_params,
@@ -817,6 +817,72 @@ pub(crate) async fn badge_handler(
817817
))
818818
}
819819

820+
#[derive(Clone, Deserialize, Debug)]
821+
pub(crate) struct JsonDownloadParams {
822+
pub(crate) name: String,
823+
pub(crate) version: ReqVersion,
824+
pub(crate) target: Option<String>,
825+
pub(crate) format_version: Option<RustdocJsonFormatVersion>,
826+
}
827+
828+
#[instrument(skip_all)]
829+
pub(crate) async fn json_download_handler(
830+
Path(params): Path<JsonDownloadParams>,
831+
mut conn: DbConnection,
832+
Extension(config): Extension<Arc<Config>>,
833+
) -> AxumResult<impl IntoResponse> {
834+
let matched_release = match_version(&mut conn, &params.name, &params.version)
835+
.await?
836+
.assume_exact_name()?;
837+
838+
if !matched_release.rustdoc_status() {
839+
// without docs we'll never have JSON docs too
840+
return Err(AxumNope::ResourceNotFound);
841+
}
842+
843+
let krate = CrateDetails::from_matched_release(&mut conn, matched_release).await?;
844+
845+
let target = if let Some(wanted_target) = params.target {
846+
if krate
847+
.metadata
848+
.doc_targets
849+
.as_ref()
850+
.expect("we are checking rustdoc_status() above, so we always have metadata")
851+
.iter()
852+
.any(|s| s == &wanted_target)
853+
{
854+
wanted_target
855+
} else {
856+
return Err(AxumNope::TargetNotFound);
857+
}
858+
} else {
859+
krate
860+
.metadata
861+
.default_target
862+
.as_ref()
863+
.expect("we are checking rustdoc_status() above, so we always have metadata")
864+
.to_string()
865+
};
866+
867+
let format_version = params
868+
.format_version
869+
.unwrap_or(RustdocJsonFormatVersion::Latest);
870+
871+
let storage_path = rustdoc_json_path(
872+
&krate.name,
873+
&krate.version.to_string(),
874+
&target,
875+
format_version,
876+
);
877+
878+
// since we didn't build rustdoc json for all releases yet,
879+
// this redirect might redirect to a location that doesn't exist.
880+
Ok(super::axum_cached_redirect(
881+
format!("{}/{}", config.s3_static_root_path, storage_path),
882+
CachePolicy::ForeverInCdn,
883+
)?)
884+
}
885+
820886
#[instrument(skip_all)]
821887
pub(crate) async fn download_handler(
822888
Path((name, req_version)): Path<(String, ReqVersion)>,
@@ -874,6 +940,7 @@ pub(crate) async fn static_asset_handler(
874940

875941
#[cfg(test)]
876942
mod test {
943+
use super::*;
877944
use crate::{
878945
Config,
879946
registry_api::{CrateOwner, OwnerKind},
@@ -3009,4 +3076,132 @@ mod test {
30093076
Ok(())
30103077
});
30113078
}
3079+
3080+
#[test_case(
3081+
"latest/json",
3082+
"0.2.0",
3083+
"x86_64-unknown-linux-gnu",
3084+
RustdocJsonFormatVersion::Latest
3085+
)]
3086+
#[test_case(
3087+
"0.1/json",
3088+
"0.1.0",
3089+
"x86_64-unknown-linux-gnu",
3090+
RustdocJsonFormatVersion::Latest;
3091+
"semver"
3092+
)]
3093+
#[test_case(
3094+
"0.1.0/json",
3095+
"0.1.0",
3096+
"x86_64-unknown-linux-gnu",
3097+
RustdocJsonFormatVersion::Latest
3098+
)]
3099+
#[test_case(
3100+
"latest/json/latest",
3101+
"0.2.0",
3102+
"x86_64-unknown-linux-gnu",
3103+
RustdocJsonFormatVersion::Latest
3104+
)]
3105+
#[test_case(
3106+
"latest/json/42",
3107+
"0.2.0",
3108+
"x86_64-unknown-linux-gnu",
3109+
RustdocJsonFormatVersion::Version(42)
3110+
)]
3111+
#[test_case(
3112+
"latest/i686-pc-windows-msvc/json",
3113+
"0.2.0",
3114+
"i686-pc-windows-msvc",
3115+
RustdocJsonFormatVersion::Latest
3116+
)]
3117+
#[test_case(
3118+
"latest/i686-pc-windows-msvc/json/42",
3119+
"0.2.0",
3120+
"i686-pc-windows-msvc",
3121+
RustdocJsonFormatVersion::Version(42)
3122+
)]
3123+
fn json_download(
3124+
request_path_suffix: &str,
3125+
redirect_version: &str,
3126+
redirect_target: &str,
3127+
redirect_format_version: RustdocJsonFormatVersion,
3128+
) {
3129+
async_wrapper(|env| async move {
3130+
env.override_config(|config| {
3131+
config.s3_static_root_path = "https://static.docs.rs".into();
3132+
});
3133+
env.fake_release()
3134+
.await
3135+
.name("dummy")
3136+
.version("0.1.0")
3137+
.archive_storage(true)
3138+
.default_target("x86_64-unknown-linux-gnu")
3139+
.add_target("i686-pc-windows-msvc")
3140+
.create()
3141+
.await?;
3142+
3143+
env.fake_release()
3144+
.await
3145+
.name("dummy")
3146+
.version("0.2.0")
3147+
.archive_storage(true)
3148+
.default_target("x86_64-unknown-linux-gnu")
3149+
.add_target("i686-pc-windows-msvc")
3150+
.create()
3151+
.await?;
3152+
3153+
let web = env.web_app().await;
3154+
3155+
web.assert_redirect_cached_unchecked(
3156+
&format!("/crate/dummy/{request_path_suffix}"),
3157+
&format!("https://static.docs.rs/rustdoc-json/dummy/{redirect_version}/{redirect_target}/\
3158+
dummy_{redirect_version}_{redirect_target}_{redirect_format_version}.json.zst"),
3159+
CachePolicy::ForeverInCdn,
3160+
&env.config(),
3161+
)
3162+
.await?;
3163+
Ok(())
3164+
});
3165+
}
3166+
3167+
#[test_case("0.1.0/json"; "rustdoc status false")]
3168+
#[test_case("0.2.0/unknown-target/json"; "unknown target")]
3169+
#[test_case("0.42.0/json"; "unknown version")]
3170+
fn json_download_not_found(request_path_suffix: &str) {
3171+
async_wrapper(|env| async move {
3172+
env.override_config(|config| {
3173+
config.s3_static_root_path = "https://static.docs.rs".into();
3174+
});
3175+
3176+
env.fake_release()
3177+
.await
3178+
.name("dummy")
3179+
.version("0.1.0")
3180+
.archive_storage(true)
3181+
.default_target("x86_64-unknown-linux-gnu")
3182+
.add_target("i686-pc-windows-msvc")
3183+
.binary(true) // binary => rustdoc_status = false
3184+
.create()
3185+
.await?;
3186+
3187+
env.fake_release()
3188+
.await
3189+
.name("dummy")
3190+
.version("0.2.0")
3191+
.archive_storage(true)
3192+
.default_target("x86_64-unknown-linux-gnu")
3193+
.add_target("i686-pc-windows-msvc")
3194+
.create()
3195+
.await?;
3196+
3197+
let web = env.web_app().await;
3198+
3199+
let response = web
3200+
.get(&format!("/crate/dummy/{request_path_suffix}"))
3201+
.await?;
3202+
3203+
assert_eq!(response.status(), StatusCode::NOT_FOUND);
3204+
Ok(())
3205+
});
3206+
}
30123207
}

‎src/web/sitemap.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ about_page!(AboutPageBadges, "core/about/badges.html");
142142
about_page!(AboutPageMetadata, "core/about/metadata.html");
143143
about_page!(AboutPageRedirection, "core/about/redirections.html");
144144
about_page!(AboutPageDownload, "core/about/download.html");
145+
about_page!(AboutPageRustdocJson, "core/about/rustdoc-json.html");
145146

146147
pub(crate) async fn about_handler(subpage: Option<Path<String>>) -> AxumResult<impl IntoResponse> {
147148
let subpage = match subpage {
@@ -170,6 +171,10 @@ pub(crate) async fn about_handler(subpage: Option<Path<String>>) -> AxumResult<i
170171
active_tab: "download",
171172
}
172173
.into_response(),
174+
"rustdoc-json" => AboutPageRustdocJson {
175+
active_tab: "rustdoc-json",
176+
}
177+
.into_response(),
173178
_ => {
174179
let msg = "This /about page does not exist. \
175180
Perhaps you are interested in <a href=\"https://github.com/rust-lang/docs.rs/tree/master/templates/core/about\">creating</a> it?";

‎templates/about-base.html‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ <h1 id="crate-title" class="no-description">Docs.rs documentation</h1>
3434
{% set text = crate::icons::IconDownload.render_solid(false, false, "") %}
3535
{% set text = "{} <span class='title'>Download</span>"|format(text) %}
3636
{% call macros::active_link(expected="download", href="/about/download", text=text) %}
37+
38+
{% set text = crate::icons::IconFileCode.render_solid(false, false, "") %}
39+
{% set text = "{} <span class='title'>Rustdoc JSON</span>"|format(text) %}
40+
{% call macros::active_link(expected="rustdoc-json", href="/about/rustdoc-json", text=text) %}
3741
</ul>
3842
</div>
3943
</div>

0 commit comments

Comments
 (0)