aede_core/

wikipedia.rs

//! Wikipedia, reached through Wikidata: the prose MusicBrainz does not hold.
//!
//! MusicBrainz answers with identifiers, dates and relationships. It never
//! answers with a sentence about the artist, because that is not what it is
//! for. The sentence exists on Wikipedia — and the way from one to the other
//! is the `wikidata` relationship MusicBrainz already returns.
//!
//! **Two requests, not one.** A Wikidata entity is the hub: it holds a
//! *sitelink* per language, and a sitelink is a page title rather than a
//! summary. So reaching a paragraph is
//!
//! ```text
//! MusicBrainz artist  →  wikidata: .../wiki/Q11649
//!                        Special:EntityData/Q11649.json   →  sitelinks.enwiki.title
//!                        en.wikipedia.org/…/summary/<title>  →  extract
//! ```
//!
//! That is why this is opt-in rather than part of every fetch: it doubles a
//! run that already takes ten minutes over a large library, and the summary
//! is the one fact here nobody needs in order to file their music.
//!
//! **Everything in this module is a decision, and none of it is a request.**
//! The two URL builders and the two extractors below are pure functions over
//! text and JSON, so what the program concludes from an answer is proven by
//! the tests beside them, and only the handing of a URL to a client library is
//! left unproven — the same split [`crate::musicbrainz`] follows.
//!
//! # The licence is not optional
//!
//! Wikipedia text is CC BY-SA. Reusing it obliges attribution, so the extract
//! is only ever produced as a [`Prose`], which cannot hold the words without
//! the page they came from and the terms they came under. There is deliberately
//! no function here that returns a bare `String` of article text.

use crate::json::Json;
use crate::sources::Prose;

/// The name this source is stored under in `sources.json`.
///
/// "wikipedia" rather than "wikidata": Wikidata is the road, the article is
/// what a reader is being shown, and the credit has to name the article.
pub const SOURCE: &str = "wikipedia";

/// The licence Wikipedia article text is under.
///
/// Named as Wikimedia names it, because a credit that renames a licence is not
/// a credit. Stored per record rather than assumed at display time: if this
/// ever changes, records fetched before the change still say what they were
/// taken under.
pub const LICENCE: &str = "CC BY-SA 4.0";

/// The name a portrait reached through Wikidata is stored under.
///
/// Not `wikipedia`, even though the same entity document is the source of
/// both: an article's prose and an entity's `P18` image are two different
/// claims, fetched to answer two different questions, and a record that
/// carried both could not be re-fetched for one without silently dropping the
/// other — `Sources::set` replaces a record whole, it does not merge fields
/// into it. Keeping them as two records under two names is what lets
/// `fetch --summaries` and `fetch --portraits` each redo their own work
/// without undoing the other's.
pub const PORTRAIT_SOURCE: &str = "wikidata";

/// How long to leave between requests.
///
/// Wikimedia's limits are far looser than MusicBrainz's, but the polite rate
/// is the one that never needs revisiting, and a run that is already paced at
/// one per second gains nothing from going faster in one leg of it.
pub const REQUEST_INTERVAL: std::time::Duration = std::time::Duration::from_millis(1100);

/// Languages to look for an article in, best first.
///
/// English last rather than first would be the friendlier default and the
/// wrong one: this list is a fallback chain, and the caller puts the reader's
/// own language at its head. English is here as the floor, because for a great
/// many artists it is the only article that exists.
pub const FALLBACK_LANGS: [&str; 1] = ["en"];

/// The Wikidata entity id inside a URL MusicBrainz returned.
///
/// MusicBrainz gives the relationship as a page URL rather than a bare id, and
/// it does not always give the same one: `www.wikidata.org/wiki/Q11649` is the
/// usual shape, but the entity URI `.../entity/Q11649` appears too. Both end
/// in the id, so the id is read from the end rather than the middle.
///
/// Returns `None` for anything that does not end in an entity id, which is the
/// honest answer for a link that leads somewhere else — a lexeme, a property,
/// or a URL that was never Wikidata at all.
pub fn entity_id(url: &str) -> Option<String> {
    let address = url
        .strip_prefix("https://")
        .or_else(|| url.strip_prefix("http://"))?;
    let (host, path) = address.split_once('/')?;
    if !matches!(host, "www.wikidata.org" | "wikidata.org") {
        return None;
    }
    let last = path
        .strip_prefix("wiki/")
        .or_else(|| path.strip_prefix("entity/"))?
        .trim_end_matches('/');
    // An entity id is `Q` and at least one digit, all digits. Checking the
    // shape is what stops `.../wiki/Special:EntityData` or a property `P31`
    // from being sent out as a lookup that cannot succeed.
    let digits = last.strip_prefix('Q')?;
    match !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit()) {
        true => Some(last.to_string()),
        false => None,
    }
}

/// Where to ask Wikidata about an entity.
///
/// `Special:EntityData` rather than the `wbgetentities` API: it is a plain
/// document at a stable address, it needs no parameters to get wrong, and it
/// is cached, which is the difference between being a polite client and being
/// a load.
pub fn entity_data_url(id: &str) -> String {
    format!("https://www.wikidata.org/wiki/Special:EntityData/{id}.json")
}

/// The Commons filename in a Wikidata entity's `image` claim (`P18`), if it
/// states one.
///
/// Mirrors the redirect handling [`article`] already does: the document may
/// answer about a different id than the one asked for, and one entity in the
/// document is no ambiguity about which that is — the same reasoning, applied
/// to a claim instead of a sitelink.
///
/// Preferred usable statements precede normal ones; deprecated statements,
/// explicit non-values and blank filenames are skipped. Ties retain source
/// order. `None` means the entity supplied no usable image statement.
pub fn portrait_file(response: &Json, id: &str) -> Option<String> {
    let entities = response.get("entities")?;
    let entity = match entities.get(id) {
        Some(entity) => entity,
        None => only_value(entities)?,
    };
    let statements = entity.get("claims")?.get("P18")?.as_arr()?;
    for rank in ["preferred", "normal"] {
        for statement in statements {
            if statement
                .get("rank")
                .and_then(Json::as_str)
                .unwrap_or("normal")
                != rank
            {
                continue;
            }
            let Some(snak) = statement.get("mainsnak") else {
                continue;
            };
            if snak
                .get("snaktype")
                .and_then(Json::as_str)
                .is_some_and(|kind| kind != "value")
            {
                continue;
            }
            if let Some(filename) = snak
                .get("datavalue")
                .and_then(|value| value.get("value"))
                .and_then(Json::as_str)
                .map(str::trim)
                .filter(|name| !name.is_empty())
            {
                return Some(filename.to_string());
            }
        }
    }
    None
}

/// Checks portrait tables before a fetch records a completed absence.
///
/// Claims and P18 are optional, but a present table must keep its documented
/// shape. Unreadable tables do not establish that an artist has no portrait.
pub fn validate_portrait(response: &Json, id: &str) -> Result<(), &'static str> {
    let entity = response
        .get("entities")
        .and_then(|entities| entities.get(id).or_else(|| only_value(entities)))
        .filter(|entity| matches!(entity, Json::Obj(_)))
        .ok_or("Wikidata returned no unambiguous portrait entity")?;
    let Some(claims) = entity.get("claims") else {
        return Ok(());
    };
    let Json::Obj(claims) = claims else {
        return Err("Wikidata returned no valid claims table");
    };
    if claims
        .get("P18")
        .is_some_and(|value| !matches!(value, Json::Arr(_)))
    {
        return Err("Wikidata returned no valid portrait statements array");
    }
    Ok(())
}

/// Where to download a Commons file, at a given width.
///
/// `Special:FilePath` rather than resolving the MD5-sharded path Commons
/// actually stores the file under: it is a stable redirect Commons maintains
/// for exactly this, and following it costs one request, not a second API call
/// to a different service to compute a hash Aède has no business working out
/// itself.
pub fn commons_file_url(filename: &str, width: u32) -> String {
    format!(
        "https://commons.wikimedia.org/wiki/Special:FilePath/{}?width={width}",
        encode_title(filename)
    )
}

/// An article on one language's Wikipedia.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Article {
    /// The language code — `en`, `fr` — which is also part of the address.
    pub lang: String,
    /// The page title, as Wikidata spells it, spaces and all.
    pub title: String,
}

/// The article to read, from a `Special:EntityData` document.
///
/// `preferred` is tried in order, so a caller can ask for the reader's own
/// language and fall back rather than choosing between "always English" and
/// "nothing".
///
/// The document is keyed by entity id at the top, and an `EntityData` response
/// may carry redirects — asking for `Q1` and being answered about `Q2` — so the
/// id asked for is looked up first and, failing that, the single entity the
/// document does carry is used. Guessing between several would be a coin toss,
/// so several is `None`.
pub fn article(response: &Json, id: &str, preferred: &[&str]) -> Option<Article> {
    let entities = response.get("entities")?;
    let entity = match entities.get(id) {
        Some(entity) => entity,
        // A redirect answers under the target's id. One entity means there is
        // no ambiguity about which that is.
        None => only_value(entities)?,
    };
    let sitelinks = entity.get("sitelinks")?;
    for lang in preferred {
        // Wikidata names the site, not the language: the English Wikipedia is
        // `enwiki`, and `enwikiquote` or `enwikisource` are different projects
        // that would answer with something other than an encyclopaedia entry.
        let title = sitelinks
            .get(&format!("{lang}wiki"))
            .and_then(|s| s.field_str("title"))
            .filter(|t| !t.trim().is_empty());
        if let Some(title) = title {
            return Some(Article {
                lang: (*lang).to_string(),
                title,
            });
        }
    }
    None
}

/// Checks the entity response before a fetch records a completed absence.
///
/// Empty sitelinks are a valid answer. A missing entity, ambiguous redirect or
/// malformed table is a failed response, not evidence that no article exists.
/// As in [`article`], a single redirected entity is accepted.
pub fn validate_entity(response: &Json, id: &str) -> Result<(), &'static str> {
    let entity = response
        .get("entities")
        .and_then(|entities| entities.get(id).or_else(|| only_value(entities)))
        .ok_or("Wikidata returned no unambiguous entity")?;
    let Some(Json::Obj(sitelinks)) = entity.get("sitelinks") else {
        return Err("Wikidata returned no valid sitelinks table");
    };
    if sitelinks.values().any(|link| {
        link.get("title")
            .and_then(Json::as_str)
            .is_none_or(|title| title.trim().is_empty())
    }) {
        return Err("Wikidata returned an unreadable sitelink");
    }
    Ok(())
}

/// Distinguishes a readable summary or genuine absence from an invalid body.
///
/// A typed standard/stub, disambiguation or main page can have no extract.
/// Older responses with a string extract and no type remain readable. An
/// unknown response shape must not erase a biography or prevent a later retry.
pub fn validate_summary(response: &Json) -> Result<(), &'static str> {
    let kind = response.get("type");
    if kind.is_some_and(|value| {
        !matches!(
            value.as_str(),
            Some("standard" | "disambiguation" | "no-extract" | "mainpage")
        )
    }) {
        return Err("Wikipedia returned no recognized summary type");
    }
    match response.get("extract") {
        Some(Json::Str(_)) => Ok(()),
        None if kind.is_some() => Ok(()),
        _ => Err("Wikipedia returned no readable summary"),
    }
}

/// Where to ask for an article's opening paragraph.
///
/// The REST summary endpoint rather than the `action=query` API: it returns the
/// lead paragraph already assembled and already plain text, where the older API
/// returns wikitext or HTML that this program would then have to strip — and
/// stripping markup to display it as fact is how a citation template ends up
/// quoted at a user.
pub fn summary_url(article: &Article) -> String {
    format!(
        "https://{}.wikipedia.org/api/rest_v1/page/summary/{}",
        article.lang,
        encode_title(&article.title)
    )
}

/// A title as it goes into a REST path.
///
/// Spaces become underscores, as every Wikipedia address does, and everything
/// outside the unreserved set is percent-encoded. Parentheses are encoded too:
/// they are legal in a path, but article titles are full of them — `Marilyn
/// Manson (band)` — and encoding them costs nothing while leaving them raw
/// depends on every intermediary agreeing they are safe.
fn encode_title(title: &str) -> String {
    let mut out = String::with_capacity(title.len());
    for byte in title.replace(' ', "_").bytes() {
        match byte {
            b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
                out.push(byte as char)
            }
            other => out.push_str(&format!("%{other:02X}")),
        }
    }
    out
}

/// The paragraph, credited, from a REST summary document.
///
/// `None` when the page holds no extract — a disambiguation page, a redirect
/// to one, a stub that is only an infobox — which is not an error: it is the
/// same "asked, and there is nothing" the rest of the layer records.
///
/// The URL stored is the article's own canonical address when the response
/// gives one, and the address that was asked otherwise. It is never left out:
/// a `Prose` cannot be built without it, which is the point of the type.
pub fn prose(response: &Json, asked: &Article) -> Option<Prose> {
    // Disambiguation and other non-article pages can still carry an extract;
    // that text is not the biography reached through the entity's sitelink.
    if response
        .get("type")
        .and_then(Json::as_str)
        .is_some_and(|kind| kind != "standard")
    {
        return None;
    }
    let text = response
        .field_str("extract")
        .map(|t| t.trim().to_string())
        .filter(|t| !t.is_empty())?;
    // The canonical page URL sits under `content_urls.desktop.page`. It is
    // preferred over the address that was asked because a title may have been
    // followed through a redirect, and a credit has to point at the article a
    // reader will actually find.
    let canonical = response
        .get("content_urls")
        .and_then(|c| c.get("desktop"))
        .and_then(|d| d.field_str("page"))
        .filter(|u| !u.is_empty());
    let lang = response
        .field_str("lang")
        .filter(|l| !l.is_empty())
        .unwrap_or_else(|| asked.lang.clone());
    Some(Prose {
        text,
        url: canonical.unwrap_or_else(|| article_url(asked)),
        lang,
        licence: LICENCE.to_string(),
    })
}

/// A readable address for an article, for when the response gives none.
fn article_url(article: &Article) -> String {
    format!(
        "https://{}.wikipedia.org/wiki/{}",
        article.lang,
        encode_title(&article.title)
    )
}

/// The only value in an object, when there is exactly one.
fn only_value(value: &Json) -> Option<&Json> {
    match value {
        Json::Obj(fields) if fields.len() == 1 => fields.values().next(),
        _ => None,
    }
}

#[cfg(test)]
#[path = "wikipedia_tests.rs"]
mod tests;