diff --git a/Cargo.lock b/Cargo.lock index 5b0e276..be9b471 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -373,6 +373,20 @@ name = "bytemuck" version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc8b54b395f2fcfbb3d90c47b01c7f444d94d05bdeb775811dec868ac3bbc26" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] [[package]] name = "byteorder" @@ -511,6 +525,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" +[[package]] +name = "coe-rs" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8f1e641542c07631228b1e0dc04b69ae3c1d58ef65d5691a439711d805c698" + [[package]] name = "color_quant" version = "1.1.0" @@ -708,6 +728,12 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +[[package]] +name = "dbgf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6ca96b45ca70b8045e0462f191bd209fcb3c3bfe8dbfb1257ada54c4dd59169" + [[package]] name = "deranged" version = "0.3.11" @@ -793,6 +819,25 @@ dependencies = [ "wio", ] +[[package]] +name = "dyn-stack" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b" +dependencies = [ + "bytemuck", + "reborrow", +] + +[[package]] +name = "dyn-stack" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf6fa63092e3ca9f602f6500fddd05502412b748c4c4682938565b44eb9e0066" +dependencies = [ + "bytemuck", +] + [[package]] name = "either" version = "1.13.0" @@ -820,6 +865,58 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "equator" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c35da53b5a021d2484a7cc49b2ac7f2d840f8236a286f84202369bd338d761ea" +dependencies = [ + "equator-macro 0.2.1", +] + +[[package]] +name = "equator" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5099e7b6f0b7431c7a1c49f75929e2777693da192784f167066977a2965767af" +dependencies = [ + "equator-macro 0.4.1", +] + +[[package]] +name = "equator-macro" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf679796c0322556351f287a51b49e48f7c4986e727b5dd78c972d30e2e16cc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "equator-macro" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5322a90066ddae2b705096eb9e10c465c0498ae93bf9bdd6437415327c88e3bb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -861,6 +958,48 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "faer" +version = "0.19.4" +source = "git+https://github.com/sarah-ek/faer-rs?rev=4f3eb7e65c69f7f7df3bdd93aa868d5666db3656#4f3eb7e65c69f7f7df3bdd93aa868d5666db3656" +dependencies = [ + "bytemuck", + "coe-rs", + "dbgf", + "dyn-stack 0.11.0", + "equator 0.4.1", + "faer-entity", + "gemm", + "generativity", + "libm", + "matrixcompare", + "matrixcompare-core", + "nano-gemm", + "npyz", + "num-complex", + "num-traits", + "paste", + "rand", + "rand_distr", + "rayon", + "reborrow", + "serde", +] + +[[package]] +name = "faer-entity" +version = "0.19.2" +source = "git+https://github.com/sarah-ek/faer-rs?rev=4f3eb7e65c69f7f7df3bdd93aa868d5666db3656#4f3eb7e65c69f7f7df3bdd93aa868d5666db3656" +dependencies = [ + "bytemuck", + "coe-rs", + "libm", + "num-complex", + "num-traits", + "pulp", + "reborrow", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -1103,6 +1242,130 @@ dependencies = [ "byteorder", ] +[[package]] +name = "gemm" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400f2ffd14e7548356236c35dc39cad6666d833a852cb8a8f3f28029359bb03" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-c32", + "gemm-c64", + "gemm-common", + "gemm-f16", + "gemm-f32", + "gemm-f64", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10dc4a6176c8452d60eac1a155b454c91c668f794151a303bf3c75ea2874812d" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2032ce2c0bb150da0256338759a6fb01ca056f6dfe28c4d14af32d7f878f6f" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-common" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fd234fc525939654f47b39325fd5f55e552ceceea9135f3aa8bdba61eabef6" +dependencies = [ + "bytemuck", + "dyn-stack 0.10.0", + "half", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp", + "raw-cpuid", + "rayon", + "seq-macro", + "sysctl", +] + +[[package]] +name = "gemm-f16" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fc3652651f96a711d46b8833e1fac27a864be4bdfa81a374055f33ddd25c0c6" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common", + "gemm-f32", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "gemm-f32" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbc51c44ae3defd207e6d9416afccb3c4af1e7cef5e4960e4c720ac4d6f998e" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f37fc86e325c2415a4d0cab8324a0c5371ec06fc7d2f9cb1636fcfc9536a8d8" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "generativity" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5881e4c3c2433fe4905bb19cfd2b5d49d4248274862b68c27c33d9ba4e13f9ec" + [[package]] name = "generic-array" version = "0.14.7" @@ -1202,8 +1465,10 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ + "bytemuck", "cfg-if", "crunchy", + "num-traits", ] [[package]] @@ -1754,6 +2019,22 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matrixcompare" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37832ba820e47c93d66b4360198dccb004b43c74abc3ac1ce1fed54e65a80445" +dependencies = [ + "matrixcompare-core", + "num-traits", +] + +[[package]] +name = "matrixcompare-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0bdabb30db18805d5290b3da7ceaccbddba795620b86c02145d688e04900a73" + [[package]] name = "matrixmultiply" version = "0.3.9" @@ -1862,6 +2143,76 @@ dependencies = [ "typenum", ] +[[package]] +name = "nano-gemm" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f563548d38f390ef9893e4883ec38c1fb312f569e98d76bededdd91a3b41a043" +dependencies = [ + "equator 0.2.2", + "nano-gemm-c32", + "nano-gemm-c64", + "nano-gemm-codegen", + "nano-gemm-core", + "nano-gemm-f32", + "nano-gemm-f64", + "num-complex", +] + +[[package]] +name = "nano-gemm-c32" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a40449e57a5713464c3a1208c4c3301c8d29ee1344711822cf022bc91373a91b" +dependencies = [ + "nano-gemm-codegen", + "nano-gemm-core", + "num-complex", +] + +[[package]] +name = "nano-gemm-c64" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743a6e6211358fba85d1009616751e4107da86f4c95b24e684ce85f25c25b3bf" +dependencies = [ + "nano-gemm-codegen", + "nano-gemm-core", + "num-complex", +] + +[[package]] +name = "nano-gemm-codegen" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "963bf7c7110d55430169dc74c67096375491ed580cd2ef84842550ac72e781fa" + +[[package]] +name = "nano-gemm-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe3fc4f83ae8861bad79dc3c016bd6b0220da5f9de302e07d3112d16efc24aa6" + +[[package]] +name = "nano-gemm-f32" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3681b7ce35658f79da94b7f62c60a005e29c373c7111ed070e3bf64546a8bb" +dependencies = [ + "nano-gemm-codegen", + "nano-gemm-core", +] + +[[package]] +name = "nano-gemm-f64" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc1e619ed04d801809e1f63e61b669d380c4119e8b0cdd6ed184c6b111f046d8" +dependencies = [ + "nano-gemm-codegen", + "nano-gemm-core", +] + [[package]] name = "native-tls" version = "0.2.12" @@ -1901,6 +2252,17 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" +[[package]] +name = "npyz" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13f27ea175875c472b3df61ece89a6d6ef4e0627f43704e400c782f174681ebd" +dependencies = [ + "byteorder", + "num-bigint", + "py_literal", +] + [[package]] name = "num" version = "0.4.3" @@ -1931,7 +2293,9 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ + "bytemuck", "num-traits", + "rand", ] [[package]] @@ -2114,6 +2478,51 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pest" +version = "2.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdbef9d1d47087a895abd220ed25eb4ad973a5e26f6a4367b038c25e28dfc2d9" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3a6e3394ec80feb3b6393c725571754c6188490265c61aaf260810d6b95aa0" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94429506bde1ca69d1b5601962c73f4172ab4726571a59ea95931218cb0e930e" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "pest_meta" +version = "2.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac8a071862e93690b6e34e9a5fb8e33ff3734473ac0245b27232222c4906a33f" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + [[package]] name = "pin-project" version = "1.1.5" @@ -2302,6 +2711,31 @@ dependencies = [ "unicase", ] +[[package]] +name = "pulp" +version = "0.18.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6" +dependencies = [ + "bytemuck", + "libm", + "num-complex", + "reborrow", +] + +[[package]] +name = "py_literal" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "102df7a3d46db9d3891f178dcc826dc270a6746277a9ae6436f8d29fd490a8e1" +dependencies = [ + "num-bigint", + "num-complex", + "num-traits", + "pest", + "pest_derive", +] + [[package]] name = "qoi" version = "0.4.1" @@ -2437,6 +2871,15 @@ dependencies = [ "rgb", ] +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -2463,6 +2906,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + [[package]] name = "redox_syscall" version = "0.5.4" @@ -2868,6 +3317,12 @@ dependencies = [ "serde", ] +[[package]] +name = "seq-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" + [[package]] name = "serde" version = "1.0.210" @@ -3035,6 +3490,7 @@ dependencies = [ "chrono", "clap", "discord-rich-presence", + "faer", "freetype-rs", "hypertesseract", "image 0.25.2", @@ -3200,6 +3656,20 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "sysctl" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" +dependencies = [ + "bitflags 2.6.0", + "byteorder", + "enum-as-inner", + "libc", + "thiserror", + "walkdir", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -3658,6 +4128,12 @@ dependencies = [ "syn 2.0.77", ] +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "unicase" version = "2.7.0" diff --git a/Cargo.toml b/Cargo.toml index 4ab2634..559fc6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,13 @@ axum = { version = "0.7.6", features = ["macros"] } paste = "1.0.15" discord-rich-presence = "0.2.4" reqwest = { version = "0.12.7", features = ["json"] } +faer = { git = "https://github.com/sarah-ek/faer-rs", rev = "4f3eb7e65c69f7f7df3bdd93aa868d5666db3656", features = ["serde"] } -# [profile.dev.package."*"] -# opt-level = 3 +[profile.dev.package.imageproc] +opt-level = 3 + +[profile.dev.package.image] +opt-level = 3 + +[profile.dev.package.faer] +opt-level = 3 diff --git a/README.md b/README.md index eea1ebd..75ad74c 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,9 @@ Additionally, you must place a custom `b30` background at `$SHIMMERING_ASSET_DIR After everything has been placed in the right directory, run `shimmeringmoon-cli prepare-jackets` to prepare everything. This will: - Associate each asset with it's database ID -- Build out a recognition matrix for image recognition purposes (this matrix more or less contains a 8x8 downscaled version of each provided asset, stored in bitmap format together with the associated database ID) +- Build out a recognition matrix (~30kb) for image recognition purposes. This file contains: + - about ~3 pixels worth of information for each jacket, stored together with their associated database IDs + - a projection matrix which transforms a $8 \times 8$ downscaled vectorized version of an image (that's $192$ dimensions — $64 \text{pixels} \times 3 \text{channels}$) and projects it to a $10$-dimensional space (the matrix is built using [singular value decomposition](https://en.wikipedia.org/wiki/Singular_value_decomposition)). ### Importing charts diff --git a/flake.nix b/flake.nix index d56310d..d07de71 100644 --- a/flake.nix +++ b/flake.nix @@ -21,7 +21,7 @@ # }; # toolchain = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.default); # toolchain = pkgs.rust-bin.stable.latest.default; - # toolchain = inputs.fenix.packages.${system}.complete.toolchain; + toolchain = inputs.fenix.packages.${system}.complete.toolchain; spkgs = inputs.self.packages.${system}; inherit (pkgs) lib; in @@ -46,11 +46,12 @@ # {{{ Devshell devShell = pkgs.mkShell rec { nativeBuildInputs = with pkgs; [ - cargo - rustc - clippy - rust-analyzer - rustfmt + # cargo + # rustc + # clippy + # rust-analyzer + # rustfmt + toolchain ruff imagemagick diff --git a/shimmering/config/shorthands.csv b/shimmering/config/shorthands.csv index 9ebcded..bf4876c 100644 --- a/shimmering/config/shorthands.csv +++ b/shimmering/config/shorthands.csv @@ -11,6 +11,7 @@ Kanbu de Tomatte Sugu Tokeru,,,overdrive 1F√,,,onefr [X],,,infinity 0xe0e1ccull,,,ifirmx +Last,,,last Last | Moment,,,last Last | Eternity,,,lasteternity Lost Emotion feat. nomico,,,lostemotion @@ -53,9 +54,11 @@ Einherjar Joker,,,einherjar GOODTEK (Arcaea Edit),,,goodtek Kanagawa Cyber Culvert,,,kanagawa La'qryma of the Wasteland,,,laqryma +PRAGMATISM,,,pragmatism PRAGMATISM -RESURRECTION-,,,pragmatism qualia -ideaesthesia-,,,qualia Shades of Light in a Transcendent Realm,,,shadesoflight trappola bewitching,,,trappola +Vicious Heroism,,,viciousheroism Vicious [ANTi] Heroism,,,viciousheroism eden,,,edenwacca diff --git a/src/arcaea/jacket.rs b/src/arcaea/jacket.rs index a5181ab..16d84bc 100644 --- a/src/arcaea/jacket.rs +++ b/src/arcaea/jacket.rs @@ -2,13 +2,13 @@ use std::fs; use anyhow::Context; -use image::{imageops::FilterType, GenericImageView, Rgba}; -use num::Integer; +use faer::{Mat, MatRef}; +use image::{GenericImageView, Pixel}; +use num::{Integer, ToPrimitive}; use serde::{Deserialize, Serialize}; -use serde_with::serde_as; use crate::arcaea::chart::{Difficulty, Jacket, SongCache}; -use crate::assets::{get_asset_dir, should_skip_jacket_art}; +use crate::assets::get_asset_dir; use crate::context::Error; // }}} @@ -16,184 +16,170 @@ use crate::context::Error; pub const SPLIT_FACTOR: u32 = 8; pub const IMAGE_VEC_DIM: usize = (SPLIT_FACTOR * SPLIT_FACTOR * 3) as usize; pub const BITMAP_IMAGE_SIZE: u32 = 174; +pub const JACKET_RECOGNITITION_DIMENSIONS: usize = 10; -#[serde_as] -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ImageVec { - #[serde_as(as = "[_; IMAGE_VEC_DIM]")] - pub colors: [f32; IMAGE_VEC_DIM], +// {{{ (Image => vector) encoding +#[allow(clippy::identity_op)] +pub fn image_to_vec(image: &impl GenericImageView) -> MVec { + let mut colors = MVec::zeros(IMAGE_VEC_DIM, 1); + let chunk_width = image.width() / SPLIT_FACTOR; + let chunk_height = image.height() / SPLIT_FACTOR; + for i in 0..(SPLIT_FACTOR * SPLIT_FACTOR) { + let (iy, ix) = i.div_rem(&SPLIT_FACTOR); + let cropped = image.view( + chunk_width * ix, + chunk_height * iy, + chunk_width, + chunk_height, + ); + + let mut r = 0; + let mut g = 0; + let mut b = 0; + let mut count = 0; + + for (_, _, pixel) in cropped.pixels() { + let channels = pixel.channels(); + + // I'm not sure this does what it's supposed to do for non rgb(a) pixels... + r += channels[0].to_u64().unwrap().pow(2); + g += channels[1].to_u64().unwrap().pow(2); + b += channels[2].to_u64().unwrap().pow(2); + + count += 1; + } + + let count = count as f64; + let r = (r as f64 / count).sqrt(); + let g = (g as f64 / count).sqrt(); + let b = (b as f64 / count).sqrt(); + colors[(i as usize * 3 + 0, 0)] = r as f32; + colors[(i as usize * 3 + 1, 0)] = g as f32; + colors[(i as usize * 3 + 2, 0)] = b as f32; + } + + colors +} +// }}} + +/// A column vector +pub type MVec = Mat; + +/// This struct holds: +/// - a set of (song_id, vec) pairs of different images projected through the +/// aforementioned transform. +/// - an projection matrix for dimensionality reduction +#[derive(Clone, Serialize, Deserialize)] +pub struct JacketCache { + /// A matrix with each column corresponding to the result of passing a jacket + /// through [[image_to_vec]], and then projecting it through `transform_matrix` + pub jacket_matrix: Mat, + + /// Assigns each column of `jacket_matrix` a song id. + pub jacket_ids: Vec, + + /// A projection matrix for dimensionality reduction. + pub transform_matrix: Mat, } -impl ImageVec { - // {{{ (Image => vector) encoding - #[allow(clippy::identity_op)] - pub fn from_image(image: &impl GenericImageView>) -> Self { - let mut colors = [0.0; IMAGE_VEC_DIM]; - let chunk_width = image.width() / SPLIT_FACTOR; - let chunk_height = image.height() / SPLIT_FACTOR; - for i in 0..(SPLIT_FACTOR * SPLIT_FACTOR) { - let (iy, ix) = i.div_rem(&SPLIT_FACTOR); - let cropped = image.view( - chunk_width * ix, - chunk_height * iy, - chunk_width, - chunk_height, - ); +// {{{ Read jackets +pub fn read_jackets(song_cache: &mut SongCache) -> Result<(), Error> { + let suffix = format!("_{BITMAP_IMAGE_SIZE}.jpg"); + let songs_dir = get_asset_dir().join("songs/by_id"); + let entries = fs::read_dir(songs_dir).with_context(|| "Couldn't read songs directory")?; - let mut r = 0; - let mut g = 0; - let mut b = 0; - let mut count = 0; + for entry in entries { + let dir = entry?; + let raw_dir_name = dir.file_name(); + let dir_name = raw_dir_name.to_str().unwrap(); + let song_id = dir_name + .parse() + .with_context(|| format!("Dir name {dir_name} could not be parsed as `u32` song id"))?; - for (_, _, pixel) in cropped.pixels() { - r += (pixel.0[0] as u64).pow(2); - g += (pixel.0[1] as u64).pow(2); - b += (pixel.0[2] as u64).pow(2); - count += 1; + let entries = fs::read_dir(dir.path()).with_context(|| "Couldn't read song directory")?; + for entry in entries { + let file = entry?; + let raw_name = file.file_name(); + let name = raw_name.to_str().unwrap(); + if !name.ends_with(&suffix) { + continue; } - let count = count as f64; - let r = (r as f64 / count).sqrt(); - let g = (g as f64 / count).sqrt(); - let b = (b as f64 / count).sqrt(); - colors[i as usize * 3 + 0] = r as f32; - colors[i as usize * 3 + 1] = g as f32; - colors[i as usize * 3 + 2] = b as f32; - } + let name = name.strip_suffix(&suffix).unwrap(); - Self { colors } - } + let difficulty = Difficulty::DIFFICULTY_SHORTHANDS + .iter() + .zip(Difficulty::DIFFICULTIES) + .find_map(|(s, d)| Some(d).filter(|_| name == s.to_lowercase())); - #[inline] - pub fn distance_squared_to(&self, other: &Self) -> f32 { - let mut total = 0.0; + let contents: &'static _ = fs::read(file.path()) + .with_context(|| "Coult not read prepared jacket image")? + .leak(); - for i in 0..IMAGE_VEC_DIM { - let d = self.colors[i] - other.colors[i]; - total += d * d; - } + let image = image::load_from_memory(contents) + .with_context(|| "Could not load jacket image from prepared bytes")?; + let bitmap: &'static _ = Box::leak(Box::new(image.into_rgb8())); - total - } - // }}} -} - -#[derive(Clone)] -pub struct JacketCache { - jackets: Vec<(u32, ImageVec)>, -} - -impl JacketCache { - // {{{ Generate - // This is a bit inefficient (using a hash set), but only runs once - pub fn new(song_cache: &mut SongCache) -> Result { - let jacket_vectors = if should_skip_jacket_art() { - let path = get_asset_dir().join("placeholder_jacket.jpg"); - let contents: &'static _ = fs::read(path)?.leak(); - let image = image::load_from_memory(contents)?; - let bitmap: &'static _ = Box::leak(Box::new( - image - .resize(BITMAP_IMAGE_SIZE, BITMAP_IMAGE_SIZE, FilterType::Nearest) - .into_rgb8(), - )); - - for chart in song_cache.charts_mut() { + if let Some(difficulty) = difficulty { + let chart = song_cache + .lookup_by_difficulty_mut(song_id, difficulty) + .unwrap(); + chart.jacket_source = Some(difficulty); chart.cached_jacket = Some(Jacket { raw: contents, bitmap, }); - } - - Vec::new() - } else { - let suffix = format!("_{BITMAP_IMAGE_SIZE}.jpg"); - let songs_dir = get_asset_dir().join("songs/by_id"); - let entries = - fs::read_dir(songs_dir).with_context(|| "Couldn't read songs directory")?; - let bytes = fs::read(get_asset_dir().join("songs/recognition_matrix")) - .with_context(|| "Could not read jacket recognition matrix")?; - let jacket_vectors = postcard::from_bytes(&bytes) - .with_context(|| "Could not decode jacket recognition matrix")?; - - for entry in entries { - let dir = entry?; - let raw_dir_name = dir.file_name(); - let dir_name = raw_dir_name.to_str().unwrap(); - let song_id = dir_name.parse().with_context(|| { - format!("Dir name {dir_name} could not be parsed as `u32` song id") - })?; - - let entries = - fs::read_dir(dir.path()).with_context(|| "Couldn't read song directory")?; - for entry in entries { - let file = entry?; - let raw_name = file.file_name(); - let name = raw_name.to_str().unwrap(); - if !name.ends_with(&suffix) { - continue; - } - - let name = name.strip_suffix(&suffix).unwrap(); - - let difficulty = Difficulty::DIFFICULTY_SHORTHANDS - .iter() - .zip(Difficulty::DIFFICULTIES) - .find_map(|(s, d)| Some(d).filter(|_| name == s.to_lowercase())); - - let contents: &'static _ = fs::read(file.path()) - .with_context(|| "Coult not read prepared jacket image")? - .leak(); - - let image = image::load_from_memory(contents) - .with_context(|| "Could not load jacket image from prepared bytes")?; - let bitmap: &'static _ = Box::leak(Box::new(image.into_rgb8())); - - if let Some(difficulty) = difficulty { - let chart = song_cache - .lookup_by_difficulty_mut(song_id, difficulty) - .unwrap(); - chart.jacket_source = Some(difficulty); + } else { + for (_, chart_id) in song_cache.lookup_song(song_id)?.charts() { + let chart = song_cache.lookup_chart_mut(chart_id)?; + if chart.jacket_source.is_none() { chart.cached_jacket = Some(Jacket { raw: contents, bitmap, }); - } else { - for (_, chart_id) in song_cache.lookup_song(song_id)?.charts() { - let chart = song_cache.lookup_chart_mut(chart_id)?; - if chart.jacket_source.is_none() { - chart.cached_jacket = Some(Jacket { - raw: contents, - bitmap, - }); - chart.jacket_source = None; - } - } + chart.jacket_source = None; } } } + } + } - jacket_vectors - }; + Ok(()) +} +// }}} - let result = Self { - jackets: jacket_vectors, - }; +impl JacketCache { + // {{{ Generate + pub fn new() -> Result { + let bytes = fs::read(get_asset_dir().join("songs/recognition_matrix")) + .with_context(|| "Could not read jacket recognition matrix")?; + + let result = postcard::from_bytes(&bytes)?; + // .with_context(|| "Could not decode jacket recognition matrix")?; Ok(result) } // }}} // {{{ Recognise + /// Transforms a vector from image space to recognition space. #[inline] - pub fn recognise( - &self, - image: &impl GenericImageView>, - ) -> Option<(f32, &u32)> { - let vec = ImageVec::from_image(image); - self.jackets + pub fn transform_vec(&self, vec: MatRef) -> MVec { + &self.transform_matrix * vec + } + + #[inline] + pub fn recognise(&self, image: &impl GenericImageView) -> Option<(f32, u32)> { + let vec = self.transform_vec(image_to_vec(image).as_ref()); + self.jacket_ids .iter() - .map(|(i, v)| (i, v, v.distance_squared_to(&vec))) - .min_by(|(_, _, d1), (_, _, d2)| d1.partial_cmp(d2).expect("NaN distance encountered")) - .map(|(i, _, d)| (d.sqrt(), i)) + .enumerate() + .map(|(idx, id)| { + (id, { + (self.jacket_matrix.subcols(idx, 1) - &vec).squared_norm_l2() + }) + }) + .min_by(|(_, d1), (_, d2)| d1.partial_cmp(d2).expect("NaN distance encountered")) + .map(|(i, d)| (d.sqrt(), *i)) } // }}} } diff --git a/src/assets.rs b/src/assets.rs index 9b47c02..e725f6c 100644 --- a/src/assets.rs +++ b/src/assets.rs @@ -71,11 +71,6 @@ pub static UNI_FONT: RefCell = get_font("unifont.otf"); } // }}} // {{{ Asset art helpers -#[inline] -pub fn should_skip_jacket_art() -> bool { - var("SHIMMERING_NO_JACKETS").unwrap_or_default() == "1" -} - #[inline] #[allow(dead_code)] pub fn should_blur_jacket_art() -> bool { diff --git a/src/bin/cli/commands/prepare_jackets.rs b/src/bin/cli/commands/prepare_jackets.rs index 17287c3..d05f80a 100644 --- a/src/bin/cli/commands/prepare_jackets.rs +++ b/src/bin/cli/commands/prepare_jackets.rs @@ -3,10 +3,14 @@ use std::fs; use std::io::{stdout, Write}; use anyhow::{anyhow, bail, Context}; +use faer::Mat; use image::imageops::FilterType; use shimmeringmoon::arcaea::chart::{Difficulty, SongCache}; -use shimmeringmoon::arcaea::jacket::{ImageVec, BITMAP_IMAGE_SIZE}; +use shimmeringmoon::arcaea::jacket::{ + image_to_vec, read_jackets, JacketCache, BITMAP_IMAGE_SIZE, IMAGE_VEC_DIM, + JACKET_RECOGNITITION_DIMENSIONS, +}; use shimmeringmoon::assets::{get_asset_dir, get_data_dir}; use shimmeringmoon::context::{connect_db, Error}; use shimmeringmoon::recognition::fuzzy_song_name::guess_chart_name; @@ -15,13 +19,17 @@ use shimmeringmoon::recognition::fuzzy_song_name::guess_chart_name; /// Hacky function which clears the current line of the standard output. #[inline] fn clear_line() { - print!("\r \r"); + print!("\r \r"); } pub fn run() -> Result<(), Error> { let db = connect_db(&get_data_dir()); - let song_cache = SongCache::new(&db)?; + let mut song_cache = SongCache::new(&db)?; + let mut jacket_vector_ids = vec![]; + let mut jacket_vectors = vec![]; + + // {{{ Prepare directories let songs_dir = get_asset_dir().join("songs"); let raw_songs_dir = songs_dir.join("raw"); @@ -30,9 +38,8 @@ pub fn run() -> Result<(), Error> { fs::remove_dir_all(&by_id_dir).with_context(|| "Could not remove `by_id` dir")?; } fs::create_dir_all(&by_id_dir).with_context(|| "Could not create `by_id` dir")?; - - let mut jacket_vectors = vec![]; - + // }}} + // {{{ Traverse raw songs directory let entries = fs::read_dir(&raw_songs_dir) .with_context(|| "Couldn't read songs directory")? .collect::, _>>() @@ -84,12 +91,7 @@ pub fn run() -> Result<(), Error> { // the same directory. To do this, we only allow the base jacket to refer // to the FUTURE difficulty, unless it's the only jacket present // (or unless we are parsing the tutorial) - let search_difficulty = - if entries.len() > 1 && difficulty.is_none() && dir_name != "tutorial" { - Some(Difficulty::FTR) - } else { - difficulty - }; + let search_difficulty = difficulty; let (song, _) = guess_chart_name(dir_name, &song_cache, search_difficulty, true) .with_context(|| format!("Could not recognise chart name from '{dir_name}'"))?; @@ -120,12 +122,12 @@ pub fn run() -> Result<(), Error> { .with_context(|| format!("Could not read image for file {:?}", file.path()))? .leak(); let image = image::load_from_memory(contents)?; - - jacket_vectors.push((song.id, ImageVec::from_image(&image))); - let small_image = image.resize(BITMAP_IMAGE_SIZE, BITMAP_IMAGE_SIZE, FilterType::Gaussian); + jacket_vector_ids.push(song.id); + jacket_vectors.push(image_to_vec(&image)); + { let image_small_path = out_dir.join(format!("{difficulty_string}_{BITMAP_IMAGE_SIZE}.jpg")); @@ -150,27 +152,100 @@ pub fn run() -> Result<(), Error> { } } } + // }}} clear_line(); + println!("Successfully processed jackets"); - // NOTE: this is N^2, but it's a one-off warning thing, so it's fine + read_jackets(&mut song_cache)?; + println!("Successfully read jackets"); + + // {{{ Warn on missing jackets for chart in song_cache.charts() { - if jacket_vectors.iter().all(|(i, _)| chart.song_id != *i) { + if chart.cached_jacket.is_none() { println!( "No jacket found for '{} [{:?}]'", - song_cache.lookup_song(chart.song_id)?.song.title, + song_cache.lookup_song(chart.song_id)?.song, chart.difficulty ) } } + println!("No missing jackets detected"); + // }}} + // {{{ Compute jacket vec matrix + let mut jacket_matrix: Mat = Mat::zeros(IMAGE_VEC_DIM, jacket_vectors.len()); + + for (i, v) in jacket_vectors.iter().enumerate() { + jacket_matrix.subcols_mut(i, 1).copy_from(v); + } + // }}} + // {{{ Compute transform matrix + let transform_matrix = { + let svd = jacket_matrix.thin_svd(); + + svd.u() + .transpose() + .submatrix(0, 0, JACKET_RECOGNITITION_DIMENSIONS, IMAGE_VEC_DIM) + .to_owned() + }; + // }}} + // {{{ Build jacket cache + let jacket_cache = JacketCache { + jacket_ids: jacket_vector_ids, + jacket_matrix: &transform_matrix * &jacket_matrix, + transform_matrix, + }; + // }}} + + // {{{ Perform jacket recognition test + let chart_count = song_cache.charts().count(); + for (i, chart) in song_cache.charts().enumerate() { + let song = &song_cache.lookup_song(chart.song_id)?.song; + + // {{{ Update console display + if i != 0 { + clear_line(); + } + + print!("{}/{}: {song}", i, chart_count); + + if i % 5 == 0 { + stdout().flush()?; + } + // }}} + + if let Some(jacket) = chart.cached_jacket { + if let Some((_, song_id)) = jacket_cache.recognise(jacket.bitmap) { + if song_id != song.id { + let mistake = &song_cache.lookup_song(song_id)?.song; + bail!( + "Could not recognise jacket for {song} [{}]. Found song {mistake} instead.", + chart.difficulty + ) + } + } else { + bail!( + "Could not recognise jacket for {song} [{}].", + chart.difficulty + ) + } + } + } + // }}} + + clear_line(); + println!("Successfully tested jacket recognition"); + + // {{{ Save recognition matrix to disk { println!("Encoded {} images", jacket_vectors.len()); - let bytes = postcard::to_allocvec(&jacket_vectors) + let bytes = postcard::to_allocvec(&jacket_cache) .with_context(|| "Coult not encode jacket matrix")?; fs::write(songs_dir.join("recognition_matrix"), bytes) .with_context(|| "Could not write jacket matrix")?; } + // }}} Ok(()) } diff --git a/src/context.rs b/src/context.rs index 3542ecf..3da426a 100644 --- a/src/context.rs +++ b/src/context.rs @@ -7,6 +7,7 @@ use std::fs; use std::path::Path; use std::sync::LazyLock; +use crate::arcaea::jacket::read_jackets; use crate::arcaea::{chart::SongCache, jacket::JacketCache}; use crate::assets::{get_data_dir, EXO_FONT, GEOSANS_FONT, KAZESAWA_BOLD_FONT, KAZESAWA_FONT}; use crate::recognition::{hyperglass::CharMeasurements, ui::UIMeasurements}; @@ -109,7 +110,10 @@ impl UserContext { let mut song_cache = SongCache::new(&db)?; let ui_measurements = UIMeasurements::read()?; - let jacket_cache = timed!("make_jacket_cache", { JacketCache::new(&mut song_cache)? }); + let jacket_cache = JacketCache::new()?; + timed!("read_jackets", { + read_jackets(&mut song_cache)?; + }); // {{{ Font measurements static WHITELIST: &str = "0123456789'abcdefghklmnopqrstuvwxyzABCDEFGHIJKLMNOPRSTUVWXYZ"; diff --git a/src/recognition/recognize.rs b/src/recognition/recognize.rs index 10fb790..7c2f964 100644 --- a/src/recognition/recognize.rs +++ b/src/recognition/recognize.rs @@ -327,7 +327,7 @@ impl ImageAnalyzer { bail!("No known jacket looks like this"); } - let (song, chart) = ctx.song_cache.lookup_by_difficulty(*song_id, difficulty)?; + let (song, chart) = ctx.song_cache.lookup_by_difficulty(song_id, difficulty)?; Ok((song, chart)) }