diff --git a/Cargo.lock b/Cargo.lock index 1d3dc9a6..2ade91c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3524,6 +3524,7 @@ dependencies = [ "scraper", "serde", "serde_json", + "strsim", "url", ] diff --git a/crates/onetagger-autotag/src/lib.rs b/crates/onetagger-autotag/src/lib.rs index 1269a0d7..b814fddf 100644 --- a/crates/onetagger-autotag/src/lib.rs +++ b/crates/onetagger-autotag/src/lib.rs @@ -169,20 +169,20 @@ impl TrackImpl for Track { tag.set_field(Field::Style, self.styles.clone(), config.overwrite_tag(SupportedTag::Style)); } } + // Release dates if config.tag_enabled(SupportedTag::ReleaseDate) { if let Some(date) = self.release_date { + // Force Month and Day to None so standard ID3 YEAR is only YYYY tag.set_date(&TagDate { year: date.year() as i32, - month: match config.only_year { - true => None, - false => Some(date.month() as u8) - }, - day: match config.only_year { - true => None, - false => Some(date.day() as u8) - } + month: None, + day: None }, config.overwrite_tag(SupportedTag::ReleaseDate)); + + // Write the full YYYY-MM-DD date to a custom RELEASETIME tag + let date_str = date.format("%Y-%m-%d").to_string(); + tag.set_raw("RELEASETIME", vec![date_str], config.overwrite_tag(SupportedTag::ReleaseDate)); } else if let Some(year) = self.release_year { tag.set_date(&TagDate { year: year as i32, @@ -191,20 +191,20 @@ impl TrackImpl for Track { }, config.overwrite_tag(SupportedTag::ReleaseDate)); } } + // Publish date if config.tag_enabled(SupportedTag::PublishDate) { if let Some(date) = self.publish_date { + // Force Month and Day to None so standard ID3 PUBLISH YEAR is only YYYY tag.set_publish_date(&TagDate { year: date.year() as i32, - month: match config.only_year { - true => None, - false => Some(date.month() as u8) - }, - day: match config.only_year { - true => None, - false => Some(date.day() as u8) - } + month: None, + day: None }, config.overwrite_tag(SupportedTag::PublishDate)); + + // Write the full YYYY-MM-DD date to a custom PUBLISHTIME tag + let date_str = date.format("%Y-%m-%d").to_string(); + tag.set_raw("PUBLISHTIME", vec![date_str], config.overwrite_tag(SupportedTag::PublishDate)); } else if let Some(year) = self.publish_year { tag.set_publish_date(&TagDate { year: year as i32, @@ -213,6 +213,7 @@ impl TrackImpl for Track { }, config.overwrite_tag(SupportedTag::PublishDate)); } } + // URL if config.tag_enabled(SupportedTag::URL) { tag.set_raw("WWWAUDIOFILE", vec![self.url.to_string()], config.overwrite_tag(SupportedTag::URL)); diff --git a/crates/onetagger-platforms/Cargo.toml b/crates/onetagger-platforms/Cargo.toml index 09d389d9..37057f16 100644 --- a/crates/onetagger-platforms/Cargo.toml +++ b/crates/onetagger-platforms/Cargo.toml @@ -31,3 +31,4 @@ rspotify = { version = "0.15", features = [ onetagger-tag = { path = "../onetagger-tag" } onetagger-shared = { path = "../onetagger-shared" } onetagger-tagger = { path = "../onetagger-tagger" } +strsim = "0.11.1" diff --git a/crates/onetagger-platforms/src/beatport.rs b/crates/onetagger-platforms/src/beatport.rs index 718d8530..2df262d9 100644 --- a/crates/onetagger-platforms/src/beatport.rs +++ b/crates/onetagger-platforms/src/beatport.rs @@ -1,9 +1,10 @@ -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::collections::{HashMap, HashSet}; use anyhow::{anyhow, Error}; use std::time::Duration; use reqwest::StatusCode; use reqwest::blocking::Client; -use chrono::NaiveDate; +use chrono::{NaiveDate, Datelike}; use serde::{Serialize, Deserialize}; use onetagger_tag::FrameName; use onetagger_tagger::{ @@ -15,13 +16,51 @@ use serde_json::json; const INVALID_ART: &'static str = "ab2d1d04-233d-4b08-8234-9782b34dcab8"; +static MIX_REGEX: OnceLock = OnceLock::new(); +static FEATURE_REGEX: OnceLock = OnceLock::new(); + +#[derive(PartialEq, Eq, Debug)] +enum MixType { + Original, + Extended, + Club, + Radio, + Edit, + Remix, + Dub, + Unknown, +} + +impl MixType { + fn from_str(s: &str) -> Self { + let m = s.to_lowercase(); + + if m.contains("remix") || m.contains("rmx") || m.contains("rework") { + MixType::Remix + } else if m.contains("dub") { + MixType::Dub + } else if m.contains("extended") { + MixType::Extended + } else if m.contains("club") { + MixType::Club + } else if m.contains("radio") { + MixType::Radio + } else if m.contains("edit") || m.contains("short") { + MixType::Edit + } else if m.is_empty() || m == "original mix" || m == "original" || m == "orig. mix" { + MixType::Original + } else { + MixType::Unknown + } + } +} + pub struct Beatport { client: Client, access_token: Arc>> } impl Beatport { - /// Create new instance pub fn new(access_token: Arc>>) -> Beatport { let client = Client::builder() .user_agent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36") @@ -35,7 +74,6 @@ impl Beatport { } } - /// Search for tracks on Beatport using the catalog API pub fn search(&self, query: &str, page: i32, results_per_page: usize) -> Result { let query = Self::clear_search_query(query); let token = self.update_token()?; @@ -55,38 +93,48 @@ impl Beatport { Ok(response) } - /// Update embed auth token pub fn update_token(&self) -> Result { - let mut token = self.access_token.lock().unwrap(); - - // Fetch new token if it does not exist - if (*token).is_none() { - // Taken from https://embed.beatport.com/?id=4418593&type=release - let mut response: BeatportOAuth = self.client.post("https://account.beatport.com/o/token/") - .form(&json!({ - "client_id": "2tiTbKxmQFwnbFjMONU4k7njMRZmV3ZMwRBndiZs", - "client_secret": "RDUJyAk4zFEGtQ8rsTmylDSfxmALRNBn3D1BsRr7MKi3oa1TL9Mq9QxqUPK7loiumXolEWbJcWa4IGAhtwnTz1cSXClGJ1tkkNCNWwRwjxIKTZJKOJxbwaNt0Rm3WG0v", - "grant_type": "client_credentials" - })) - .send()? - .json()?; - - response.expires_in = response.expires_in * 1000 + timestamp!() - 10_000; - *token = Some(response); - debug!("OAuth: {:?}", token); - } + // BOUNDED RETRY LOOP: Prevents infinite recursion if the Beatport API + // continually serves expired tokens (e.g., due to severe clock skew). + for attempt in 1..=2 { + let mut token = self.access_token.lock().unwrap(); + + if (*token).is_none() { + let mut response: BeatportOAuth = self.client.post("https://account.beatport.com/o/token/") + .form(&json!({ + "client_id": "2tiTbKxmQFwnbFjMONU4k7njMRZmV3ZMwRBndiZs", + "client_secret": "RDUJyAk4zFEGtQ8rsTmylDSfxmALRNBn3D1BsRr7MKi3oa1TL9Mq9QxqUPK7loiumXolEWbJcWa4IGAhtwnTz1cSXClGJ1tkkNCNWwRwjxIKTZJKOJxbwaNt0Rm3WG0v", + "grant_type": "client_credentials" + })) + .send()? + .json()?; + + response.expires_in = response.expires_in * 1000 + timestamp!() - 10_000; + *token = Some(response); + debug!("OAuth: {:?}", token); + } + + let t = token.clone().unwrap(); + + // If the token is valid, return it immediately. + if t.expires_in > timestamp!() { + return Ok(t.access_token); + } - // Refresh if expired - let t = token.clone().unwrap(); - if t.expires_in <= timestamp!() { + // If we are here, the token registered as instantly expired. + if attempt == 2 { + return Err(anyhow!("Beatport API continuously provided an expired token. Please check your system clock sync.")); + } + + // Clear the token to force a fresh fetch on the next iteration. *token = None; - return self.update_token(); + // The Mutex guard (`token`) is automatically and safely dropped here at the end of the loop scope. } - Ok(t.access_token) + Err(anyhow!("Failed to acquire a valid Beatport token.")) } - /// Fetch track using API + pub fn track(&self, id: i64) -> Result, Error> { let token = self.update_token()?; @@ -95,7 +143,6 @@ impl Beatport { .bearer_auth(token) .send()?; - // Restricted / deleted track if response.status() == StatusCode::FORBIDDEN { return Ok(None); } @@ -103,7 +150,6 @@ impl Beatport { Ok(response.json()?) } - /// Fetch release using API pub fn release(&self, id: i64) -> Result { let token = self.update_token()?; @@ -116,7 +162,6 @@ impl Beatport { Ok(response) } - /// Get tracks from release pub fn release_tracks(&self, id: i64) -> Result, Error> { let token = self.update_token()?; @@ -129,26 +174,25 @@ impl Beatport { Ok(response.results) } - /// Beatport returns 403 if you have more than a single () pair pub fn clear_search_query(query: &str) -> String { - let mut open = 0; - let mut closed = 0; - - query.chars().filter(|c| { - match c { - '(' if open > 0 => false, - '(' => { - open += 1; - true - }, - ')' if closed > 0 => false, - ')' => { - closed += 1; - true - }, - _ => true - } - }).collect() + let re = FEATURE_REGEX.get_or_init(|| regex::Regex::new(r"(?i)\s+(?:ft|feat|featuring)\.?\s+[^()]+").unwrap()); + let stripped_query = re.replace_all(query, "").to_string(); + + stripped_query + .replace("(", " ") + .replace(")", " ") + .replace("[", " ") + .replace("]", " ") + .replace(",", " ") + .replace("Ft.", "") + .replace("ft.", "") + .replace(" Ft ", " ") + .replace(" ft ", " ") + .replace(" feat. ", " ") + .replace(" feat ", " ") + .replace(" ", " ") + .trim() + .to_string() } } @@ -158,7 +202,6 @@ pub struct BeatportOAuth { pub expires_in: u128 } -/// When searching for tracks #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BeatportTrackResults { #[serde(rename = "tracks")] @@ -196,7 +239,7 @@ pub struct BeatportGeneric { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BeatportImage { - pub id: i64, + pub id: Option, pub dynamic_uri: String, } @@ -255,7 +298,6 @@ impl BeatportTrack { ..Default::default() }; - // Exclusive if self.exclusive { track.other.push((FrameName::same("BEATPORT_EXCLUSIVE"), vec!["1".to_string()])); } @@ -263,7 +305,6 @@ impl BeatportTrack { track } - /// Get album art URL pub fn get_art(&self, art_resolution: u32) -> Option { if self.release.image.dynamic_uri.contains(&INVALID_ART) { return None; @@ -281,14 +322,11 @@ impl BeatportTrack { } } -// Match track impl AutotaggerSource for Beatport { fn match_track(&mut self, info: &AudioFileInfo, config: &TaggerConfig) -> Result, Error> { - // Load custom config let custom_config: BeatportConfig = config.get_custom("beatport")?; let mut output = vec![]; - // Fetch by ID if let Some(id) = info.tags.get("BEATPORT_TRACK_ID").map(|t| t.first().and_then(|id| id.trim().replace("\0", "").parse().ok())).flatten() { info!("Fetching by ID: {}", id); @@ -301,79 +339,181 @@ impl AutotaggerSource for Beatport { output.push(track); }, Ok(None) => warn!("Matching by ID failed, track restricted, matching normally"), - Err(e) => { - warn!("Matching by ID failed, matching normally: {e}"); - } + Err(e) => warn!("Matching by ID failed, matching normally: {e}") } } - // Fetch by ISRC if let Some(isrc) = info.isrc.as_ref() { match self.search(isrc, 1, 25) { Ok(results) => { if !results.data.is_empty() { - let track = self.track(results.data[0].id)?; - - match track { - Some(track) => { + // FIX: Graceful failover if track detail fetch fails, rather than hard crashing `?` + match self.track(results.data[0].id) { + Ok(Some(track)) => { let track = TrackMatch::new_isrc(track.to_track(custom_config.art_resolution)); if !config.fetch_all_results { return Ok(vec![track]); } output.push(track); }, - None => warn!("Matching by ISRC failed, track restricted, trying normal."), + Ok(None) => warn!("Matching by ISRC failed, track restricted, trying normal."), + Err(e) => warn!("Failed fetching track details by ISRC: {e}, falling back to normal match."), } } }, - Err(e) => { - warn!("Failed fetching track by ISRC: {e}"); - }, + Err(e) => warn!("Failed fetching track by ISRC: {e}"), } } - // Search - let query = format!("{} {}", info.artist()?, MatchingUtils::clean_title(info.title()?)); + let raw_title = info.title().unwrap_or(""); + let re_feat = FEATURE_REGEX.get_or_init(|| regex::Regex::new(r"(?i)\s+(?:ft|feat|featuring)\.?\s+[^()]+").unwrap()); + let virtual_title = re_feat.replace_all(raw_title, "").to_string(); + + let query = format!("{} {}", info.artist()?, MatchingUtils::clean_title(&virtual_title)); debug!("BP Query: {}", query); for page in 1..custom_config.max_pages + 1 { match self.search(&query, page, 50) { Ok(res) => { - // Match - let tracks = res.data + let api_tracks = res.data .into_iter() .map(|t| t.to_track(custom_config.art_resolution)) .collect::>(); - // Ignore version match - let tracks = if custom_config.ignore_version { - let t = tracks.clone().into_iter().map(|mut t| { + let mut matched_tracks = if custom_config.ignore_version { + let t = api_tracks.clone().into_iter().map(|mut t| { t.version = None; t }).collect(); - // Copy back versions MatchingUtils::match_track(info, &t, config, true) .into_iter() .map(|mut t| { - if let Some(ot) = tracks.iter().find(|ot| ot.url == t.track.url) { + if let Some(ot) = api_tracks.iter().find(|ot| ot.url == t.track.url) { t.track.version = ot.version.to_owned(); } t }) .collect() } else { - MatchingUtils::match_track(info, &tracks, config, true) + MatchingUtils::match_track(info, &api_tracks, config, true) }; - // Return - output.extend(tracks); + // --- NEW FALLBACK LOGIC START --- + let local_title_full = virtual_title.clone(); + + let re_mix = MIX_REGEX.get_or_init(|| regex::Regex::new(r"^(.*?)\s*(?:\(|\[)([^()\[\]]+)(?:\)|\])$").unwrap()); + + let (local_title, local_mix) = if let Some(caps) = re_mix.captures(&local_title_full) { + (caps.get(1).map_or("", |m| m.as_str()).trim(), caps.get(2).map_or("", |m| m.as_str()).trim()) + } else { + (local_title_full.trim(), "") + }; + + let normalize_punctuation = |s: &str| -> String { + s.to_lowercase().replace(['(', ')', '[', ']', '-', ',', '.', '!'], " ") + }; + + let normalize_artists = |artists: &Vec| -> HashSet { + artists.iter() + .flat_map(|a| a.to_lowercase().replace("&", "and").split(',').map(|s| s.trim().to_string()).collect::>()) + .filter(|s| !s.is_empty()) + .collect() + }; + + let local_mix_clean = normalize_punctuation(local_mix); + let local_cat = MixType::from_str(&local_mix_clean); + + for track in &api_tracks { + let beatport_title_original = &track.title; + let beatport_mix_clean = normalize_punctuation(track.version.as_deref().unwrap_or("")); + let api_cat = MixType::from_str(&beatport_mix_clean); + + let mut is_compatible = false; + let mut mix_acc = 1.0; + + if local_cat == api_cat + || (local_cat == MixType::Club && api_cat == MixType::Extended) + || (local_cat == MixType::Extended && api_cat == MixType::Club) { + + if local_cat == MixType::Remix || local_cat == MixType::Dub { + let stopwords = ["remix", "rmx", "mix", "edit", "version", "vip", "rework", "dub"]; + let words_local: HashSet<&str> = local_mix_clean.split_whitespace().filter(|w| !stopwords.contains(w)).collect(); + let words_api: HashSet<&str> = beatport_mix_clean.split_whitespace().filter(|w| !stopwords.contains(w)).collect(); + let intersection = words_local.intersection(&words_api).count() as f64; + let union = words_local.union(&words_api).count() as f64; + let jaccard = if union == 0.0 { 1.0 } else { intersection / union }; + + if jaccard >= 0.50 { + is_compatible = true; + mix_acc = jaccard; + } + } else { + is_compatible = true; + } + } else if (local_cat == MixType::Original && api_cat == MixType::Unknown) || + (local_cat == MixType::Unknown && api_cat == MixType::Original) || + (local_cat == MixType::Unknown && api_cat == MixType::Unknown) { + is_compatible = true; + mix_acc = 0.9; + } + + if !is_compatible { + continue; + } + + let beatport_title_clean = re_feat.replace_all(beatport_title_original, "").to_string(); + + let title_acc = strsim::normalized_levenshtein( + &MatchingUtils::clean_title_matching(&local_title), + &MatchingUtils::clean_title_matching(&beatport_title_clean) + ); + + let artist_acc = if MatchingUtils::match_artist(&info.artists, &track.artists, config.strictness) { + 1.0 + } else { + let api_artist_words = normalize_artists(&track.artists); + let mut local_artists_rescued = normalize_artists(&info.artists); + let beatport_title_original_lower = beatport_title_original.to_lowercase(); + + for api_artist in &api_artist_words { + if beatport_title_original_lower.contains(api_artist) { + local_artists_rescued.insert(api_artist.clone()); + } + } + + let intersection = local_artists_rescued.intersection(&api_artist_words).count() as f64; + let union = local_artists_rescued.union(&api_artist_words).count() as f64; + + if union == 0.0 { 0.0 } else { intersection / union } + }; + + mix_acc = mix_acc.clamp(0.0, 1.0); + let final_acc = (title_acc * 0.5) + (artist_acc * 0.3) + (mix_acc * 0.2); + + if final_acc >= config.strictness { + if let Some(existing) = matched_tracks.iter_mut().find(|m| m.track.url == track.url) { + if final_acc > existing.accuracy { + existing.accuracy = final_acc; + } + } else { + matched_tracks.push(TrackMatch::new(final_acc, track.clone())); + } + } + } + + matched_tracks.sort_by(|a, b| b.accuracy.partial_cmp(&a.accuracy).unwrap_or(std::cmp::Ordering::Equal)); + // --- NEW FALLBACK LOGIC END --- + + output.extend(matched_tracks); if config.fetch_all_results { continue; } - - return Ok(output); + + if output.iter().any(|m| m.accuracy >= config.strictness) { + return Ok(output); + } }, Err(e) => { warn!("Beatport search failed, query: {}. {}", query, e); @@ -388,23 +528,31 @@ impl AutotaggerSource for Beatport { fn extend_track(&mut self, track: &mut Track, config: &TaggerConfig) -> Result<(), Error> { let custom_config: BeatportConfig = config.get_custom("beatport")?; - // Extend search results track if track.other.is_empty() { - let id = track.track_id.as_ref().unwrap().parse().unwrap(); - *track = self.track(id)?.ok_or(anyhow!("Restricted track"))?.to_track(custom_config.art_resolution); + if let Some(track_id_str) = &track.track_id { + if let Ok(id) = track_id_str.parse() { + if let Ok(Some(api_track)) = self.track(id) { + *track = api_track.to_track(custom_config.art_resolution); + } + } + } } - // Ignore extending track if !config.tag_enabled(SupportedTag::AlbumArtist) && !config.tag_enabled(SupportedTag::TrackTotal) { return Ok(()); } - let release = self.release(track.release_id.as_ref().ok_or(anyhow!("Missing release_id"))?.parse()?)?; - track.track_total = release.track_count; - track.album_artists = match release.artists { - Some(a) => a.into_iter().map(|a| a.name).collect(), - None => vec![], - }; + if let Some(release_id_str) = &track.release_id { + if let Ok(id) = release_id_str.parse() { + if let Ok(release) = self.release(id) { + track.track_total = release.track_count; + track.album_artists = match release.artists { + Some(a) => a.into_iter().map(|a| a.name).collect(), + None => vec![], + }; + } + } + } Ok(()) } @@ -425,7 +573,6 @@ impl AutotaggerSource for Beatport { } } -/// For creating Beatport instances #[derive(Debug, Clone)] pub struct BeatportBuilder { access_token: Arc>> @@ -458,21 +605,18 @@ impl AutotaggerSourceBuilder for BeatportBuilder { ISRC, TrackNumber ), custom_options: PlatformCustomOptions::new() - // Album art resolution .add("art_resolution", "Album art resolution", PlatformCustomOptionValue::Number { min: 200, max: 1600, step: 100, value: 500 }) - // Max pages to search .add_tooltip("max_pages", "Max pages", "How many pages of search results to scan for tracks", PlatformCustomOptionValue::Number { min: 1, max: 10, step: 1, value: 1 }) - // Ignore version .add_tooltip("ignore_version", "Ignore version when matching", "Ignores (Extended Mix), (Original Mix) and such", PlatformCustomOptionValue::Boolean { value: false }) @@ -496,4 +640,4 @@ fn test_album() { let mut bp = builder.get_source(&config).unwrap(); bp.get_album("2174307", &config).unwrap(); -} \ No newline at end of file +} diff --git a/crates/onetagger-tag/src/id3.rs b/crates/onetagger-tag/src/id3.rs index 299f99f8..49beab0e 100644 --- a/crates/onetagger-tag/src/id3.rs +++ b/crates/onetagger-tag/src/id3.rs @@ -395,11 +395,15 @@ impl TagImpl for ID3Tag { // TXXX if tag.len() != 4 { if overwrite || self.get_raw(tag).is_none() { + + // FIX: Remove existing frame FIRST to prevent duplicates + self.tag.remove_extended_text(Some(tag), None); + // Remove if empty if value.is_empty() { - self.tag.remove_extended_text(Some(tag), None); - return; + return; } + self.tag.add_frame(ExtendedText { description: tag.to_string(), value: value.join(&self.id3_separator),