From 8298bdf7cbf6ba38a7f300e6757d4d4d2fb05bac Mon Sep 17 00:00:00 2001 From: prescientmoon Date: Thu, 8 Aug 2024 17:37:46 +0200 Subject: [PATCH] New UI measurement system Signed-off-by: prescientmoon --- data/ui.txt | 27 +- src/chart.rs | 10 +- src/commands/score.rs | 58 +-- src/context.rs | 2 +- src/jacket.rs | 2 +- src/ocr/mod.rs | 2 +- src/ocr/{ui_interp.rs => ui.rs} | 19 +- src/score.rs | 623 ++++---------------------------- 8 files changed, 142 insertions(+), 601 deletions(-) rename src/ocr/{ui_interp.rs => ui.rs} (89%) diff --git a/data/ui.txt b/data/ui.txt index 65eed86..10b42b3 100644 --- a/data/ui.txt +++ b/data/ui.txt @@ -1,11 +1,28 @@ -2160 1620 +2532 1170 Inksurgence + 237 16 273 60 Play kind +1037 462 476 91 Score screen — score + 274 434 614 611 Score screen — jacket + 378 332 161 34 Score screen — difficulty +1288 849 82 39 Score screen — pures +1288 909 82 39 Score screen — fars +1288 969 82 39 Score screen — losts + 584 377 74 31 Score screen — max recall + 634 116 1252 102 Score screen — title + 95 256 278 49 Song select — score + 465 319 730 45 Song select — jacket + 89 153 0 0 Song select — PST + 269 153 0 0 Song select — PRS + 452 153 0 0 Song select — FTR + 638 153 0 0 Song select — ETR/BYD + +2160 1620 prescientmoon 19 15 273 60 Play kind - 841 683 500 92 Score screen — score + 841 682 500 94 Score screen — score 51 655 633 632 Score screen — jacket 155 546 167 38 Score screen — difficulty -1095 1087 87 34 Score screen — pures -1095 1150 87 34 Score screen — fars -1095 1212 87 34 Score screen — losts +1104 1087 87 34 Score screen — pures +1104 1150 87 34 Score screen — fars +1104 1212 87 34 Score screen — losts 364 593 87 34 Score screen — max recall 438 324 1244 104 Score screen — title 15 264 291 52 Song select — score diff --git a/src/chart.rs b/src/chart.rs index ad49f64..74e8421 100644 --- a/src/chart.rs +++ b/src/chart.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use image::{ImageBuffer, Rgb}; use sqlx::SqlitePool; -use crate::context::Error; +use crate::{bitmap::Color, context::Error}; // {{{ Difficuly #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sqlx::Type)] @@ -42,6 +42,14 @@ impl TryFrom for Difficulty { Err(format!("Cannot convert {} to difficulty", value)) } } + +pub const DIFFICULTY_MENU_PIXEL_COLORS: [Color; Difficulty::DIFFICULTIES.len()] = [ + Color::from_rgb_int(0xAAE5F7), + Color::from_rgb_int(0xBFDD85), + Color::from_rgb_int(0xCB74AB), + Color::from_rgb_int(0xC4B7D3), + Color::from_rgb_int(0xF89AAC), +]; // }}} // {{{ Side #[derive(Debug, Clone, Copy)] diff --git a/src/commands/score.rs b/src/commands/score.rs index 1ec6374..bc048b3 100644 --- a/src/commands/score.rs +++ b/src/commands/score.rs @@ -75,8 +75,8 @@ pub async fn magic( if let Some(_) = file.dimensions() { // {{{ Image pre-processing let bytes = file.download().await?; - let image = image::load_from_memory(&bytes)?; - let mut image = image.resize(1024, 1024, FilterType::Nearest); + let mut image = image::load_from_memory(&bytes)?; + // image = image.resize(1024, 1024, FilterType::Nearest); // }}} // {{{ Detection // Create cropper and run OCR @@ -96,7 +96,7 @@ pub async fn magic( .content(format!("Image {}: reading kind", i + 1)); handle.edit(ctx, edited).await?; - let kind = match cropper.read_score_kind(&ocr_image) { + let kind = match cropper.read_score_kind(ctx.data(), &ocr_image) { // {{{ OCR error handling Err(err) => { error_with_image( @@ -121,7 +121,7 @@ pub async fn magic( handle.edit(ctx, edited).await?; // Do not use `ocr_image` because this reads the colors - let difficulty = match cropper.read_difficulty(&image, kind) { + let difficulty = match cropper.read_difficulty(ctx.data(), &image, kind) { // {{{ OCR error handling Err(err) => { error_with_image( @@ -146,9 +146,13 @@ pub async fn magic( let song_by_jacket = cropper .read_jacket(ctx.data(), &mut image, kind, difficulty, &mut jacket_rect) .await; - let note_distribution = cropper.read_distribution(&image)?; - // }}} + // image.invert(); ocr_image.invert(); + let note_distribution = match kind { + ScoreKind::ScoreScreen => Some(cropper.read_distribution(ctx.data(), &image)?), + ScoreKind::SongSelect => None, + }; + // }}} // {{{ Title let edited = CreateReply::default() .reply(true) @@ -158,7 +162,7 @@ pub async fn magic( let song_by_name = match kind { ScoreKind::SongSelect => None, ScoreKind::ScoreScreen => { - Some(cropper.read_song(&ocr_image, &ctx.data().song_cache, difficulty)) + Some(cropper.read_song(ctx.data(), &ocr_image, difficulty)) } }; @@ -237,29 +241,33 @@ Title error: {:?} .content(format!("Image {}: reading score", i + 1)); handle.edit(ctx, edited).await?; - let score_possibilities = - match cropper.read_score(Some(chart.note_count), &ocr_image, kind) { - // {{{ OCR error handling - Err(err) => { - error_with_image( - ctx, - &cropper.bytes, - &file.filename, - "Could not read score from picture", - &err, - ) - .await?; + let score_possibilities = match cropper.read_score( + ctx.data(), + Some(chart.note_count), + &ocr_image, + kind, + ) { + // {{{ OCR error handling + Err(err) => { + error_with_image( + ctx, + &cropper.bytes, + &file.filename, + "Could not read score from picture", + &err, + ) + .await?; - continue; - } - // }}} - Ok(scores) => scores, - }; + continue; + } + // }}} + Ok(scores) => scores, + }; // }}} // {{{ Build play let (score, maybe_fars, score_warning) = Score::resolve_ambiguities( score_possibilities, - Some(note_distribution), + note_distribution, chart.note_count, ) .map_err(|err| { diff --git a/src/context.rs b/src/context.rs index 8be2029..974c349 100644 --- a/src/context.rs +++ b/src/context.rs @@ -2,7 +2,7 @@ use std::{fs, path::PathBuf}; use sqlx::SqlitePool; -use crate::{chart::SongCache, jacket::JacketCache, ocr::ui_interp::UIMeasurements}; +use crate::{chart::SongCache, jacket::JacketCache, ocr::ui::UIMeasurements}; // Types used by all command functions pub type Error = Box; diff --git a/src/jacket.rs b/src/jacket.rs index 719bc53..747739a 100644 --- a/src/jacket.rs +++ b/src/jacket.rs @@ -220,7 +220,7 @@ impl JacketCache { .iter() .map(|(i, v)| (i, v, v.distance_squared_to(&vec))) .min_by(|(_, _, d1), (_, _, d2)| d1.partial_cmp(d2).expect("NaN distance encountered")) - .map(|(i, _, d)| (d, i)) + .map(|(i, _, d)| (d.sqrt(), i)) } // }}} } diff --git a/src/ocr/mod.rs b/src/ocr/mod.rs index 495cbe7..6bae95d 100644 --- a/src/ocr/mod.rs +++ b/src/ocr/mod.rs @@ -1 +1 @@ -pub mod ui_interp; +pub mod ui; diff --git a/src/ocr/ui_interp.rs b/src/ocr/ui.rs similarity index 89% rename from src/ocr/ui_interp.rs rename to src/ocr/ui.rs index 1965c2d..3a9a670 100644 --- a/src/ocr/ui_interp.rs +++ b/src/ocr/ui.rs @@ -2,7 +2,9 @@ use std::{fs, path::PathBuf}; -use crate::context::Error; +use image::GenericImage; + +use crate::{bitmap::Rect, context::Error}; // {{{ Rects #[derive(Debug, Clone, Copy)] @@ -60,6 +62,7 @@ impl UIMeasurementRect { pub const UI_RECT_COUNT: usize = 15; // }}} // {{{ Measurement +#[derive(Debug)] pub struct UIMeasurement { dimensions: [u32; 2], datapoints: [u32; UI_RECT_COUNT * 4], @@ -86,6 +89,7 @@ impl UIMeasurement { } // }}} // {{{ Measurements +#[derive(Debug)] pub struct UIMeasurements { pub measurements: Vec, } @@ -106,7 +110,7 @@ impl UIMeasurements { for (j, str) in line.split_whitespace().enumerate().take(2) { measurement.dimensions[j] = u32::from_str_radix(str, 10)?; } - } else if i == UI_RECT_COUNT + 2 { + } else if i == UI_RECT_COUNT + 1 { measurements.push(measurement); measurement = UIMeasurement::default(); } else { @@ -117,7 +121,6 @@ impl UIMeasurements { } // }}} - measurements.push(measurement); measurements.sort_by_key(|r| (r.aspect_ratio() * 1000.0) as u32); // {{{ Filter datapoints that are close together @@ -135,6 +138,7 @@ impl UIMeasurements { } // }}} + println!("Read {} UI measurements", measurements.len()); Ok(Self { measurements }) } // }}} @@ -142,9 +146,9 @@ impl UIMeasurements { pub fn interpolate( &self, rect: UIMeasurementRect, - dimensions: [u32; 2], - ) -> Result<[u32; 4], Error> { - let aspect_ratio = dimensions[0] as f32 / dimensions[1] as f32; + image: &impl GenericImage, + ) -> Result { + let aspect_ratio = image.width() as f32 / image.height() as f32; let r = rect.to_index(); for i in 0..(self.measurements.len() - 1) { @@ -157,6 +161,7 @@ impl UIMeasurements { if (i == 0 || low_ratio <= aspect_ratio) && (aspect_ratio <= high_ratio || i == self.measurements.len() - 2) { + let dimensions = [image.width(), image.height()]; let p = (aspect_ratio - low_ratio) / (high_ratio - low_ratio); let mut out = [0; 4]; for j in 0..4 { @@ -165,7 +170,7 @@ impl UIMeasurements { out[j] = ((l + (h - l) * p) * dimensions[j % 2] as f32) as u32; } - return Ok(out); + return Ok(Rect::new(out[0] as i32, out[1] as i32, out[2], out[3])); } } diff --git a/src/score.rs b/src/score.rs index a4255df..8b44321 100644 --- a/src/score.rs +++ b/src/score.rs @@ -3,7 +3,6 @@ use std::fmt::Display; use std::fs; use std::io::Cursor; use std::str::FromStr; -use std::sync::OnceLock; use image::{imageops::FilterType, DynamicImage, GenericImageView}; use num::integer::Roots; @@ -15,19 +14,14 @@ use sqlx::{query_as, SqlitePool}; use tesseract::{PageSegMode, Tesseract}; use crate::bitmap::{Color, Rect}; -use crate::chart::{Chart, Difficulty, Song, SongCache}; +use crate::chart::{Chart, Difficulty, Song, SongCache, DIFFICULTY_MENU_PIXEL_COLORS}; use crate::context::{Error, UserContext}; use crate::image::rotate; use crate::jacket::IMAGE_VEC_DIM; use crate::levenshtein::{edit_distance, edit_distance_with}; +use crate::ocr::ui::{ScoreScreenRect, SongSelectRect, UIMeasurementRect}; use crate::user::User; -// {{{ Utils -#[inline] -fn lerp(i: f32, a: f32, b: f32) -> f32 { - a + (b - a) * i -} -// }}} // {{{ Grade #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum Grade { @@ -716,491 +710,6 @@ pub enum ScoreKind { ScoreScreen, } // }}} -// {{{ Image processing helpers -// {{{ ImageDimensions -#[derive(Debug, Clone, Copy)] -pub struct ImageDimensions { - width: u32, - height: u32, -} - -impl ImageDimensions { - #[inline] - pub fn new(width: u32, height: u32) -> Self { - Self { width, height } - } - - #[inline] - pub fn aspect_ratio(&self) -> f32 { - self.width as f32 / self.height as f32 - } - - #[inline] - pub fn from_image(image: &DynamicImage) -> Self { - Self::new(image.width(), image.height()) - } -} -// }}} -// {{{ AbsoluteRect -#[derive(Debug, Clone, Copy)] -pub struct AbsoluteRect { - pub x: u32, - pub y: u32, - pub width: u32, - pub height: u32, - pub dimensions: ImageDimensions, -} - -impl AbsoluteRect { - #[inline] - pub fn new(x: u32, y: u32, width: u32, height: u32, dimensions: ImageDimensions) -> Self { - Self { - x, - y, - width, - height, - dimensions, - } - } - - #[inline] - pub fn to_relative(&self) -> RelativeRect { - RelativeRect::new( - self.x as f32 / self.dimensions.width as f32, - self.y as f32 / self.dimensions.height as f32, - self.width as f32 / self.dimensions.width as f32, - self.height as f32 / self.dimensions.height as f32, - self.dimensions, - ) - } - - #[inline] - pub fn to_rect(&self) -> Rect { - Rect::new(self.x as i32, self.y as i32, self.width, self.height) - } -} -// }}} -// {{{ RelativeRect -#[derive(Debug, Clone, Copy)] -pub struct RelativeRect { - pub x: f32, - pub y: f32, - pub width: f32, - pub height: f32, - pub dimensions: ImageDimensions, -} - -impl RelativeRect { - #[inline] - pub fn new(x: f32, y: f32, width: f32, height: f32, dimensions: ImageDimensions) -> Self { - Self { - x, - y, - width, - height, - dimensions, - } - } - - /// Shift this rect on the y axis by a given absolute pixel amount - #[inline] - pub fn shift_y_abs(&self, amount: u32) -> Self { - let mut res = Self::new( - self.x, - self.y + (amount as f32 / self.dimensions.height as f32), - self.width, - self.height, - self.dimensions, - ); - res.fix(); - res - } - - /// Clamps the values apropriately - #[inline] - pub fn fix(&mut self) { - self.x = self.x.max(0.); - self.y = self.y.max(0.); - self.width = self.width.min(1. - self.x); - self.height = self.height.min(1. - self.y); - } - - #[inline] - pub fn to_absolute(&self) -> AbsoluteRect { - AbsoluteRect::new( - (self.x * self.dimensions.width as f32) as u32, - (self.y * self.dimensions.height as f32) as u32, - (self.width * self.dimensions.width as f32) as u32, - (self.height * self.dimensions.height as f32) as u32, - self.dimensions, - ) - } -} -// }}} -// {{{ AbsolutePoint -#[derive(Debug, Clone, Copy)] -pub struct AbsolutePoint { - pub x: u32, - pub y: u32, - pub dimensions: ImageDimensions, -} - -impl AbsolutePoint { - #[inline] - pub fn new(x: u32, y: u32, dimensions: ImageDimensions) -> Self { - Self { x, y, dimensions } - } - - #[inline] - pub fn to_relative(&self) -> RelativePoint { - RelativePoint::new( - self.x as f32 / self.dimensions.width as f32, - self.y as f32 / self.dimensions.height as f32, - self.dimensions, - ) - } -} -// }}} -// {{{ RelativePoint -#[derive(Debug, Clone, Copy)] -pub struct RelativePoint { - pub x: f32, - pub y: f32, - pub dimensions: ImageDimensions, -} - -impl RelativePoint { - #[inline] - pub fn new(x: f32, y: f32, dimensions: ImageDimensions) -> Self { - Self { x, y, dimensions } - } - - #[inline] - pub fn to_absolute(&self) -> AbsolutePoint { - AbsolutePoint::new( - (self.x * self.dimensions.width as f32) as u32, - (self.y * self.dimensions.height as f32) as u32, - self.dimensions, - ) - } -} -// }}} -// }}} -// {{{ Data points -// {{{ Trait -trait UIDataPoint: Sized + Copy { - fn aspect_ratio(&self) -> f32; - fn lerp(low: &Self, high: &Self, p: f32, dimensions: ImageDimensions) -> Self; - fn from_aspect_ratio(dimensions: ImageDimensions, datapoints: &[Self]) -> Option { - let aspect_ratio = dimensions.aspect_ratio(); - - for i in 0..(datapoints.len() - 1) { - let low = datapoints[i]; - let high = datapoints[i + 1]; - - let low_ratio = low.aspect_ratio(); - let high_ratio = high.aspect_ratio(); - - if (i == 0 || low_ratio <= aspect_ratio) - && (aspect_ratio <= high_ratio || i == datapoints.len() - 2) - { - let p = (aspect_ratio - low_ratio) / (high_ratio - low_ratio); - return Some(Self::lerp(&low, &high, p, dimensions)); - } - } - - None - } -} - -impl UIDataPoint for RelativeRect { - fn aspect_ratio(&self) -> f32 { - self.dimensions.aspect_ratio() - } - - fn lerp(low: &Self, high: &Self, p: f32, dimensions: ImageDimensions) -> Self { - Self::new( - lerp(p, low.x, high.x), - lerp(p, low.y, high.y), - lerp(p, low.width, high.width), - lerp(p, low.height, high.height), - dimensions, - ) - } -} - -impl UIDataPoint for RelativePoint { - fn aspect_ratio(&self) -> f32 { - self.dimensions.aspect_ratio() - } - - fn lerp(low: &Self, high: &Self, p: f32, dimensions: ImageDimensions) -> Self { - Self::new(lerp(p, low.x, high.x), lerp(p, low.y, high.y), dimensions) - } -} -// }}} -// {{{ Processing -fn process_datapoints(points: &mut Vec) { - points.sort_by_key(|r| (r.aspect_ratio() * 1000.0).floor() as u32); - - // Filter datapoints that are close together - let mut i = 0; - while i < points.len() - 1 { - let low = &points[i]; - let high = &points[i + 1]; - - if (low.aspect_ratio() - high.aspect_ratio()).abs() < 0.001 { - // TODO: we could interpolate here but oh well - points.remove(i + 1); - } - - i += 1; - } -} - -fn widen_by(rects: &mut Vec, x: f32, y: f32) { - for rect in rects { - rect.x -= x; - rect.y -= y; - rect.width += 2. * x; - rect.height += 2. * y; - rect.fix(); - } -} -// }}} -// {{{ Score (score screen) -fn score_score_screen_rects() -> &'static [RelativeRect] { - static CELL: OnceLock> = OnceLock::new(); - CELL.get_or_init(|| { - let mut rects: Vec = vec![ - AbsoluteRect::new(642, 287, 284, 51, ImageDimensions::new(1560, 720)).to_relative(), - AbsoluteRect::new(651, 285, 305, 55, ImageDimensions::new(1600, 720)).to_relative(), - AbsoluteRect::new(748, 485, 503, 82, ImageDimensions::new(2000, 1200)).to_relative(), - AbsoluteRect::new(841, 683, 500, 92, ImageDimensions::new(2160, 1620)).to_relative(), - AbsoluteRect::new(851, 707, 532, 91, ImageDimensions::new(2224, 1668)).to_relative(), - AbsoluteRect::new(1037, 462, 476, 89, ImageDimensions::new(2532, 1170)).to_relative(), - AbsoluteRect::new(973, 653, 620, 105, ImageDimensions::new(2560, 1600)).to_relative(), - AbsoluteRect::new(1069, 868, 636, 112, ImageDimensions::new(2732, 2048)).to_relative(), - AbsoluteRect::new(1125, 510, 534, 93, ImageDimensions::new(2778, 1284)).to_relative(), - ]; - process_datapoints(&mut rects); - widen_by(&mut rects, 0.0, 0.0075); - rects - }) -} -// }}} -// {{{ Score (song select) -pub fn score_song_select_rects() -> &'static [RelativeRect] { - static CELL: OnceLock> = OnceLock::new(); - CELL.get_or_init(|| { - let mut rects: Vec = vec![ - AbsoluteRect::new(95, 256, 278, 49, ImageDimensions::new(2532, 1170)).to_relative(), - AbsoluteRect::new(15, 264, 291, 52, ImageDimensions::new(2160, 1620)).to_relative(), - ]; - process_datapoints(&mut rects); - rects - }) -} -// }}} -// {{{ Difficulty -fn difficulty_rects() -> &'static [RelativeRect] { - static CELL: OnceLock> = OnceLock::new(); - CELL.get_or_init(|| { - let mut rects: Vec = vec![ - AbsoluteRect::new(232, 203, 104, 23, ImageDimensions::new(1560, 720)).to_relative(), - AbsoluteRect::new(252, 204, 99, 21, ImageDimensions::new(1600, 720)).to_relative(), - AbsoluteRect::new(146, 356, 155, 34, ImageDimensions::new(2000, 1200)).to_relative(), - AbsoluteRect::new(155, 546, 167, 38, ImageDimensions::new(2160, 1620)).to_relative(), - AbsoluteRect::new(163, 562, 175, 38, ImageDimensions::new(2224, 1668)).to_relative(), - AbsoluteRect::new(378, 332, 161, 34, ImageDimensions::new(2532, 1170)).to_relative(), - AbsoluteRect::new(183, 487, 197, 44, ImageDimensions::new(2560, 1600)).to_relative(), - AbsoluteRect::new(198, 692, 219, 46, ImageDimensions::new(2732, 2048)).to_relative(), - AbsoluteRect::new(414, 364, 177, 38, ImageDimensions::new(2778, 1284)).to_relative(), - AbsoluteRect::new(76, 172, 77, 18, ImageDimensions::new(1080, 607)).to_relative(), - ]; - process_datapoints(&mut rects); - rects - }) -} -// }}} -// {{{ Chart title -fn title_rects() -> &'static [RelativeRect] { - static CELL: OnceLock> = OnceLock::new(); - CELL.get_or_init(|| { - let mut rects: Vec = vec![ - AbsoluteRect::new(227, 74, 900, 61, ImageDimensions::new(1560, 720)).to_relative(), - AbsoluteRect::new(413, 72, 696, 58, ImageDimensions::new(1600, 720)).to_relative(), - AbsoluteRect::new(484, 148, 1046, 96, ImageDimensions::new(2000, 1200)).to_relative(), - AbsoluteRect::new(438, 324, 1244, 104, ImageDimensions::new(2160, 1620)).to_relative(), - AbsoluteRect::new(216, 336, 1366, 96, ImageDimensions::new(2224, 1668)).to_relative(), - AbsoluteRect::new(634, 116, 1252, 102, ImageDimensions::new(2532, 1170)).to_relative(), - AbsoluteRect::new(586, 222, 1320, 118, ImageDimensions::new(2560, 1600)).to_relative(), - AbsoluteRect::new(348, 417, 1716, 120, ImageDimensions::new(2732, 2048)).to_relative(), - AbsoluteRect::new(760, 128, 1270, 118, ImageDimensions::new(2778, 1284)).to_relative(), - ]; - process_datapoints(&mut rects); - widen_by(&mut rects, 0.3, 0.0); - rects - }) -} -// }}} -// {{{ Jacket (score screen) -pub fn jacket_score_screen_rects() -> &'static [RelativeRect] { - static CELL: OnceLock> = OnceLock::new(); - CELL.get_or_init(|| { - let mut rects: Vec = vec![ - AbsoluteRect::new(171, 268, 375, 376, ImageDimensions::new(1560, 720)).to_relative(), - AbsoluteRect::new(190, 267, 376, 377, ImageDimensions::new(1600, 720)).to_relative(), - AbsoluteRect::new(46, 456, 590, 585, ImageDimensions::new(2000, 1200)).to_relative(), - AbsoluteRect::new(51, 655, 633, 632, ImageDimensions::new(2160, 1620)).to_relative(), - AbsoluteRect::new(53, 675, 654, 653, ImageDimensions::new(2224, 1668)).to_relative(), - AbsoluteRect::new(274, 434, 614, 611, ImageDimensions::new(2532, 1170)).to_relative(), - AbsoluteRect::new(58, 617, 753, 750, ImageDimensions::new(2560, 1600)).to_relative(), - AbsoluteRect::new(65, 829, 799, 800, ImageDimensions::new(2732, 2048)).to_relative(), - AbsoluteRect::new(300, 497, 670, 670, ImageDimensions::new(2778, 1284)).to_relative(), - ]; - process_datapoints(&mut rects); - rects - }) -} -// }}} -// {{{ Jacket (song select) -pub fn jacket_song_select_rects() -> &'static [RelativeRect] { - static CELL: OnceLock> = OnceLock::new(); - CELL.get_or_init(|| { - let mut rects: Vec = vec![ - AbsoluteRect::new(465, 319, 730, 45, ImageDimensions::new(2532, 1170)).to_relative(), - AbsoluteRect::new(158, 411, 909, 74, ImageDimensions::new(2160, 1620)).to_relative(), - ]; - process_datapoints(&mut rects); - rects - }) -} -// }}} -// {{{ Note distribution -pub fn note_distribution_rects() -> ( - &'static [RelativeRect], - &'static [RelativeRect], - &'static [RelativeRect], -) { - static CELL: OnceLock<( - &'static [RelativeRect], - &'static [RelativeRect], - &'static [RelativeRect], - )> = OnceLock::new(); - *CELL.get_or_init(|| { - let mut pure_rects: Vec = vec![ - AbsoluteRect::new(729, 523, 58, 22, ImageDimensions::new(1560, 720)).to_relative(), - AbsoluteRect::new(815, 520, 57, 23, ImageDimensions::new(1600, 720)).to_relative(), - AbsoluteRect::new(1019, 856, 91, 33, ImageDimensions::new(2000, 1200)).to_relative(), - AbsoluteRect::new(1100, 1085, 102, 38, ImageDimensions::new(2160, 1620)).to_relative(), - AbsoluteRect::new(1130, 1118, 105, 39, ImageDimensions::new(2224, 1668)).to_relative(), - AbsoluteRect::new(1286, 850, 91, 35, ImageDimensions::new(2532, 1170)).to_relative(), - AbsoluteRect::new(1305, 1125, 117, 44, ImageDimensions::new(2560, 1600)).to_relative(), - AbsoluteRect::new(1389, 1374, 126, 48, ImageDimensions::new(2732, 2048)).to_relative(), - AbsoluteRect::new(1407, 933, 106, 40, ImageDimensions::new(2778, 1284)).to_relative(), - ]; - - process_datapoints(&mut pure_rects); - - let skip_distances = vec![40, 40, 57, 67, 65, 60, 75, 78, 65]; - let far_rects: Vec<_> = pure_rects - .iter() - .enumerate() - .map(|(i, rect)| rect.shift_y_abs(skip_distances[i])) - .collect(); - - let lost_rects: Vec<_> = far_rects - .iter() - .enumerate() - .map(|(i, rect)| rect.shift_y_abs(skip_distances[i])) - .collect(); - - (pure_rects.leak(), far_rects.leak(), lost_rects.leak()) - }) -} -// }}} -// {{{ Score kind -fn score_kind_rects() -> &'static [RelativeRect] { - static CELL: OnceLock> = OnceLock::new(); - CELL.get_or_init(|| { - let mut rects: Vec = vec![ - AbsoluteRect::new(237, 16, 273, 60, ImageDimensions::new(2532, 1170)).to_relative(), - AbsoluteRect::new(19, 15, 273, 60, ImageDimensions::new(2160, 1620)).to_relative(), - ]; - process_datapoints(&mut rects); - rects - }) -} -// }}} -// {{{ Difficulty pixel locations -fn pst_pixel() -> &'static [RelativePoint] { - static CELL: OnceLock> = OnceLock::new(); - CELL.get_or_init(|| { - let mut points: Vec = vec![ - AbsolutePoint::new(89, 153, ImageDimensions::new(2532, 1170)).to_relative(), - AbsolutePoint::new(12, 159, ImageDimensions::new(2160, 1620)).to_relative(), - ]; - process_datapoints(&mut points); - points - }) -} - -fn prs_pixel() -> &'static [RelativePoint] { - static CELL: OnceLock> = OnceLock::new(); - CELL.get_or_init(|| { - let mut points: Vec = vec![ - AbsolutePoint::new(269, 153, ImageDimensions::new(2532, 1170)).to_relative(), - AbsolutePoint::new(199, 159, ImageDimensions::new(2160, 1620)).to_relative(), - ]; - process_datapoints(&mut points); - points - }) -} - -fn ftr_pixel() -> &'static [RelativePoint] { - static CELL: OnceLock> = OnceLock::new(); - CELL.get_or_init(|| { - let mut points: Vec = vec![ - AbsolutePoint::new(452, 153, ImageDimensions::new(2532, 1170)).to_relative(), - AbsolutePoint::new(389, 159, ImageDimensions::new(2160, 1620)).to_relative(), - ]; - process_datapoints(&mut points); - points - }) -} - -fn byd_etr_pixel() -> &'static [RelativePoint] { - static CELL: OnceLock> = OnceLock::new(); - CELL.get_or_init(|| { - let mut points: Vec = vec![ - AbsolutePoint::new(638, 153, ImageDimensions::new(2532, 1170)).to_relative(), - AbsolutePoint::new(579, 159, ImageDimensions::new(2160, 1620)).to_relative(), - ]; - process_datapoints(&mut points); - points - }) -} - -fn difficulty_pixel(difficulty: Difficulty) -> &'static [RelativePoint] { - match difficulty { - Difficulty::PST => pst_pixel(), - Difficulty::PRS => prs_pixel(), - Difficulty::FTR => ftr_pixel(), - Difficulty::ETR => byd_etr_pixel(), - Difficulty::BYD => byd_etr_pixel(), - } -} - -const DIFFICULTY_MENU_PIXEL_COLORS: [Color; Difficulty::DIFFICULTIES.len()] = [ - Color::from_rgb_int(0xAAE5F7), - Color::from_rgb_int(0xBFDD85), - Color::from_rgb_int(0xCB74AB), - Color::from_rgb_int(0xC4B7D3), - Color::from_rgb_int(0xF89AAC), -]; -// }}} -// }}} // {{{ Recognise chart fn strip_case_insensitive_suffix<'a>(string: &'a str, suffix: &str) -> Option<&'a str> { let suffix = suffix.to_lowercase(); @@ -1348,6 +857,7 @@ impl ImageCropper { // {{{ Read score pub fn read_score( &mut self, + ctx: &UserContext, note_count: Option, image: &DynamicImage, kind: ScoreKind, @@ -1355,17 +865,14 @@ impl ImageCropper { println!("kind {kind:?}"); self.crop_image_to_bytes( &image.resize_exact(image.width(), image.height(), FilterType::Nearest), - RelativeRect::from_aspect_ratio( - ImageDimensions::from_image(image), + ctx.ui_measurements.interpolate( if kind == ScoreKind::ScoreScreen { - score_score_screen_rects() + UIMeasurementRect::ScoreScreen(ScoreScreenRect::Score) } else { - score_song_select_rects() + UIMeasurementRect::SongSelect(SongSelectRect::Score) }, - ) - .ok_or_else(|| "Could not find score area in picture")? - .to_absolute() - .to_rect(), + image, + )?, )?; let mut results = vec![]; @@ -1488,28 +995,32 @@ impl ImageCropper { // {{{ Read difficulty pub fn read_difficulty( &mut self, + ctx: &UserContext, image: &DynamicImage, kind: ScoreKind, ) -> Result { if kind == ScoreKind::SongSelect { - let dimensions = ImageDimensions::from_image(image); - let min = DIFFICULTY_MENU_PIXEL_COLORS .iter() .zip(Difficulty::DIFFICULTIES) .min_by_key(|(c, d)| { - let points = difficulty_pixel(*d); - let point = RelativePoint::from_aspect_ratio(dimensions, points) - .ok_or_else(|| "Could not find difficulty pixel in picture") - // SAFETY: should I just throwkkk here? - .unwrap_or(RelativePoint::new(0.0, 0.0, dimensions)) - .to_absolute(); + let rect = ctx + .ui_measurements + .interpolate( + UIMeasurementRect::SongSelect(match d { + Difficulty::PST => SongSelectRect::Past, + Difficulty::PRS => SongSelectRect::Present, + Difficulty::FTR => SongSelectRect::Future, + _ => SongSelectRect::Beyond, + }), + image, + ) + .unwrap(); - let image_color = image.get_pixel(point.x, point.y); + let image_color = image.get_pixel(rect.x as u32, rect.y as u32); let image_color = Color::from_bytes(image_color.0); let distance = c.distance(image_color); - println!("distance {distance} image_color {image_color:?} color {c:?} difficulty {d:?}"); (distance * 10000.0) as u32 }) .unwrap(); @@ -1518,11 +1029,11 @@ impl ImageCropper { } self.crop_image_to_bytes( - &image, - RelativeRect::from_aspect_ratio(ImageDimensions::from_image(image), difficulty_rects()) - .ok_or_else(|| "Could not find difficulty area in picture")? - .to_absolute() - .to_rect(), + image, + ctx.ui_measurements.interpolate( + UIMeasurementRect::ScoreScreen(ScoreScreenRect::Difficulty), + image, + )?, )?; let mut t = Tesseract::new(None, Some("eng"))?.set_image_from_mem(&self.bytes)?; @@ -1551,13 +1062,15 @@ impl ImageCropper { } // }}} // {{{ Read score kind - pub fn read_score_kind(&mut self, image: &DynamicImage) -> Result { + pub fn read_score_kind( + &mut self, + ctx: &UserContext, + image: &DynamicImage, + ) -> Result { self.crop_image_to_bytes( &image, - RelativeRect::from_aspect_ratio(ImageDimensions::from_image(image), score_kind_rects()) - .ok_or_else(|| "Could not find score kind area in picture")? - .to_absolute() - .to_rect(), + ctx.ui_measurements + .interpolate(UIMeasurementRect::PlayKind, image)?, )?; let mut t = Tesseract::new(None, Some("eng"))?.set_image_from_mem(&self.bytes)?; @@ -1587,16 +1100,16 @@ impl ImageCropper { // {{{ Read song pub fn read_song<'a>( &mut self, + ctx: &'a UserContext, image: &DynamicImage, - cache: &'a SongCache, difficulty: Difficulty, ) -> Result<(&'a Song, &'a Chart), Error> { self.crop_image_to_bytes( &image, - RelativeRect::from_aspect_ratio(ImageDimensions::from_image(image), title_rects()) - .ok_or_else(|| "Could not find title area in picture")? - .to_absolute() - .to_rect(), + ctx.ui_measurements.interpolate( + UIMeasurementRect::ScoreScreen(ScoreScreenRect::Title), + image, + )?, )?; let mut t = Tesseract::new(None, Some("eng"))? @@ -1619,7 +1132,7 @@ impl ImageCropper { // ))?; // } - guess_chart_name(raw_text, cache, Some(difficulty), false) + guess_chart_name(raw_text, &ctx.song_cache, Some(difficulty), false) } // }}} // {{{ Read jacket @@ -1631,39 +1144,32 @@ impl ImageCropper { difficulty: Difficulty, out_rect: &mut Option, ) -> Result<(&'a Song, &'a Chart), Error> { - let rect = RelativeRect::from_aspect_ratio( - ImageDimensions::from_image(image), + let rect = ctx.ui_measurements.interpolate( if kind == ScoreKind::ScoreScreen { - jacket_score_screen_rects() + UIMeasurementRect::ScoreScreen(ScoreScreenRect::Jacket) } else { - jacket_song_select_rects() + UIMeasurementRect::SongSelect(SongSelectRect::Jacket) }, - ) - .ok_or_else(|| "Could not find jacket area in picture")? - .to_absolute(); + image, + )?; let cropped = if kind == ScoreKind::ScoreScreen { - *out_rect = Some(rect.to_rect()); - image.view(rect.x, rect.y, rect.width, rect.height) + *out_rect = Some(rect); + image.view(rect.x as u32, rect.y as u32, rect.width, rect.height) } else { let angle = f32::atan2(rect.height as f32, rect.width as f32); let side = rect.height + rect.width; rotate( image, - Rect::new(rect.x as i32, rect.y as i32, side, side), - (rect.x as i32, (rect.y + rect.height) as i32), + Rect::new(rect.x, rect.y, side, side), + (rect.x, rect.y + rect.height as i32), angle, ); let len = (rect.width.pow(2) + rect.height.pow(2)).sqrt(); - *out_rect = Some(Rect::new( - rect.x as i32, - (rect.y + rect.height) as i32, - len, - len, - )); - image.view(rect.x, rect.y + rect.height, len, len) + *out_rect = Some(Rect::new(rect.x, rect.y + rect.height as i32, len, len)); + image.view(rect.x as u32, rect.y as u32 + rect.height, len, len) }; let (distance, song_id) = ctx .jacket_cache @@ -1682,19 +1188,20 @@ impl ImageCropper { } // }}} // {{{ Read distribution - pub fn read_distribution(&mut self, image: &DynamicImage) -> Result<(u32, u32, u32), Error> { + pub fn read_distribution( + &mut self, + ctx: &UserContext, + image: &DynamicImage, + ) -> Result<(u32, u32, u32), Error> { let mut t = Tesseract::new(None, Some("eng"))? .set_variable("classify_bln_numeric_mode", "1")? .set_variable("tessedit_char_whitelist", "0123456789")?; - t.set_page_seg_mode(PageSegMode::PsmSingleLine); + t.set_page_seg_mode(PageSegMode::PsmSparseText); - let (pure_rects, far_rects, lost_rects) = note_distribution_rects(); self.crop_image_to_bytes( &image, - RelativeRect::from_aspect_ratio(ImageDimensions::from_image(image), pure_rects) - .ok_or_else(|| "Could not find pure-rect area in picture")? - .to_absolute() - .to_rect(), + ctx.ui_measurements + .interpolate(UIMeasurementRect::ScoreScreen(ScoreScreenRect::Pure), image)?, )?; t = t.set_image_from_mem(&self.bytes)?.recognize()?; @@ -1703,10 +1210,8 @@ impl ImageCropper { self.crop_image_to_bytes( &image, - RelativeRect::from_aspect_ratio(ImageDimensions::from_image(image), far_rects) - .ok_or_else(|| "Could not find far-rect area in picture")? - .to_absolute() - .to_rect(), + ctx.ui_measurements + .interpolate(UIMeasurementRect::ScoreScreen(ScoreScreenRect::Far), image)?, )?; t = t.set_image_from_mem(&self.bytes)?.recognize()?; @@ -1715,10 +1220,8 @@ impl ImageCropper { self.crop_image_to_bytes( &image, - RelativeRect::from_aspect_ratio(ImageDimensions::from_image(image), lost_rects) - .ok_or_else(|| "Could not find lost-rect area in picture")? - .to_absolute() - .to_rect(), + ctx.ui_measurements + .interpolate(UIMeasurementRect::ScoreScreen(ScoreScreenRect::Lost), image)?, )?; t = t.set_image_from_mem(&self.bytes)?.recognize()?;