1
Fork 0

Use svd for dimensional reduction

This commit is contained in:
prescientmoon 2024-10-05 00:44:54 +02:00
parent 238d81f3bd
commit 645b6ef525
Signed by: prescientmoon
SSH key fingerprint: SHA256:WFp/cO76nbarETAoQcQXuV+0h7XJsEsOCI0UsyPIy6U
10 changed files with 735 additions and 186 deletions

476
Cargo.lock generated
View file

@ -373,6 +373,20 @@ name = "bytemuck"
version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae"
dependencies = [
"bytemuck_derive",
]
[[package]]
name = "bytemuck_derive"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0cc8b54b395f2fcfbb3d90c47b01c7f444d94d05bdeb775811dec868ac3bbc26"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.77",
]
[[package]]
name = "byteorder"
@ -511,6 +525,12 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15"
[[package]]
name = "coe-rs"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e8f1e641542c07631228b1e0dc04b69ae3c1d58ef65d5691a439711d805c698"
[[package]]
name = "color_quant"
version = "1.1.0"
@ -708,6 +728,12 @@ version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
[[package]]
name = "dbgf"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6ca96b45ca70b8045e0462f191bd209fcb3c3bfe8dbfb1257ada54c4dd59169"
[[package]]
name = "deranged"
version = "0.3.11"
@ -793,6 +819,25 @@ dependencies = [
"wio",
]
[[package]]
name = "dyn-stack"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b"
dependencies = [
"bytemuck",
"reborrow",
]
[[package]]
name = "dyn-stack"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf6fa63092e3ca9f602f6500fddd05502412b748c4c4682938565b44eb9e0066"
dependencies = [
"bytemuck",
]
[[package]]
name = "either"
version = "1.13.0"
@ -820,6 +865,58 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "enum-as-inner"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.77",
]
[[package]]
name = "equator"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c35da53b5a021d2484a7cc49b2ac7f2d840f8236a286f84202369bd338d761ea"
dependencies = [
"equator-macro 0.2.1",
]
[[package]]
name = "equator"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5099e7b6f0b7431c7a1c49f75929e2777693da192784f167066977a2965767af"
dependencies = [
"equator-macro 0.4.1",
]
[[package]]
name = "equator-macro"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3bf679796c0322556351f287a51b49e48f7c4986e727b5dd78c972d30e2e16cc"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.77",
]
[[package]]
name = "equator-macro"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5322a90066ddae2b705096eb9e10c465c0498ae93bf9bdd6437415327c88e3bb"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.77",
]
[[package]]
name = "equivalent"
version = "1.0.1"
@ -861,6 +958,48 @@ dependencies = [
"zune-inflate",
]
[[package]]
name = "faer"
version = "0.19.4"
source = "git+https://github.com/sarah-ek/faer-rs?rev=4f3eb7e65c69f7f7df3bdd93aa868d5666db3656#4f3eb7e65c69f7f7df3bdd93aa868d5666db3656"
dependencies = [
"bytemuck",
"coe-rs",
"dbgf",
"dyn-stack 0.11.0",
"equator 0.4.1",
"faer-entity",
"gemm",
"generativity",
"libm",
"matrixcompare",
"matrixcompare-core",
"nano-gemm",
"npyz",
"num-complex",
"num-traits",
"paste",
"rand",
"rand_distr",
"rayon",
"reborrow",
"serde",
]
[[package]]
name = "faer-entity"
version = "0.19.2"
source = "git+https://github.com/sarah-ek/faer-rs?rev=4f3eb7e65c69f7f7df3bdd93aa868d5666db3656#4f3eb7e65c69f7f7df3bdd93aa868d5666db3656"
dependencies = [
"bytemuck",
"coe-rs",
"libm",
"num-complex",
"num-traits",
"pulp",
"reborrow",
]
[[package]]
name = "fallible-iterator"
version = "0.3.0"
@ -1103,6 +1242,130 @@ dependencies = [
"byteorder",
]
[[package]]
name = "gemm"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e400f2ffd14e7548356236c35dc39cad6666d833a852cb8a8f3f28029359bb03"
dependencies = [
"dyn-stack 0.10.0",
"gemm-c32",
"gemm-c64",
"gemm-common",
"gemm-f16",
"gemm-f32",
"gemm-f64",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"seq-macro",
]
[[package]]
name = "gemm-c32"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10dc4a6176c8452d60eac1a155b454c91c668f794151a303bf3c75ea2874812d"
dependencies = [
"dyn-stack 0.10.0",
"gemm-common",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"seq-macro",
]
[[package]]
name = "gemm-c64"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc2032ce2c0bb150da0256338759a6fb01ca056f6dfe28c4d14af32d7f878f6f"
dependencies = [
"dyn-stack 0.10.0",
"gemm-common",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"seq-macro",
]
[[package]]
name = "gemm-common"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90fd234fc525939654f47b39325fd5f55e552ceceea9135f3aa8bdba61eabef6"
dependencies = [
"bytemuck",
"dyn-stack 0.10.0",
"half",
"num-complex",
"num-traits",
"once_cell",
"paste",
"pulp",
"raw-cpuid",
"rayon",
"seq-macro",
"sysctl",
]
[[package]]
name = "gemm-f16"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fc3652651f96a711d46b8833e1fac27a864be4bdfa81a374055f33ddd25c0c6"
dependencies = [
"dyn-stack 0.10.0",
"gemm-common",
"gemm-f32",
"half",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"rayon",
"seq-macro",
]
[[package]]
name = "gemm-f32"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acbc51c44ae3defd207e6d9416afccb3c4af1e7cef5e4960e4c720ac4d6f998e"
dependencies = [
"dyn-stack 0.10.0",
"gemm-common",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"seq-macro",
]
[[package]]
name = "gemm-f64"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f37fc86e325c2415a4d0cab8324a0c5371ec06fc7d2f9cb1636fcfc9536a8d8"
dependencies = [
"dyn-stack 0.10.0",
"gemm-common",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"seq-macro",
]
[[package]]
name = "generativity"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5881e4c3c2433fe4905bb19cfd2b5d49d4248274862b68c27c33d9ba4e13f9ec"
[[package]]
name = "generic-array"
version = "0.14.7"
@ -1202,8 +1465,10 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888"
dependencies = [
"bytemuck",
"cfg-if",
"crunchy",
"num-traits",
]
[[package]]
@ -1754,6 +2019,22 @@ version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
[[package]]
name = "matrixcompare"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37832ba820e47c93d66b4360198dccb004b43c74abc3ac1ce1fed54e65a80445"
dependencies = [
"matrixcompare-core",
"num-traits",
]
[[package]]
name = "matrixcompare-core"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0bdabb30db18805d5290b3da7ceaccbddba795620b86c02145d688e04900a73"
[[package]]
name = "matrixmultiply"
version = "0.3.9"
@ -1862,6 +2143,76 @@ dependencies = [
"typenum",
]
[[package]]
name = "nano-gemm"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f563548d38f390ef9893e4883ec38c1fb312f569e98d76bededdd91a3b41a043"
dependencies = [
"equator 0.2.2",
"nano-gemm-c32",
"nano-gemm-c64",
"nano-gemm-codegen",
"nano-gemm-core",
"nano-gemm-f32",
"nano-gemm-f64",
"num-complex",
]
[[package]]
name = "nano-gemm-c32"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a40449e57a5713464c3a1208c4c3301c8d29ee1344711822cf022bc91373a91b"
dependencies = [
"nano-gemm-codegen",
"nano-gemm-core",
"num-complex",
]
[[package]]
name = "nano-gemm-c64"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "743a6e6211358fba85d1009616751e4107da86f4c95b24e684ce85f25c25b3bf"
dependencies = [
"nano-gemm-codegen",
"nano-gemm-core",
"num-complex",
]
[[package]]
name = "nano-gemm-codegen"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "963bf7c7110d55430169dc74c67096375491ed580cd2ef84842550ac72e781fa"
[[package]]
name = "nano-gemm-core"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe3fc4f83ae8861bad79dc3c016bd6b0220da5f9de302e07d3112d16efc24aa6"
[[package]]
name = "nano-gemm-f32"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e3681b7ce35658f79da94b7f62c60a005e29c373c7111ed070e3bf64546a8bb"
dependencies = [
"nano-gemm-codegen",
"nano-gemm-core",
]
[[package]]
name = "nano-gemm-f64"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc1e619ed04d801809e1f63e61b669d380c4119e8b0cdd6ed184c6b111f046d8"
dependencies = [
"nano-gemm-codegen",
"nano-gemm-core",
]
[[package]]
name = "native-tls"
version = "0.2.12"
@ -1901,6 +2252,17 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8"
[[package]]
name = "npyz"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13f27ea175875c472b3df61ece89a6d6ef4e0627f43704e400c782f174681ebd"
dependencies = [
"byteorder",
"num-bigint",
"py_literal",
]
[[package]]
name = "num"
version = "0.4.3"
@ -1931,7 +2293,9 @@ version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
dependencies = [
"bytemuck",
"num-traits",
"rand",
]
[[package]]
@ -2114,6 +2478,51 @@ version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
[[package]]
name = "pest"
version = "2.7.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdbef9d1d47087a895abd220ed25eb4ad973a5e26f6a4367b038c25e28dfc2d9"
dependencies = [
"memchr",
"thiserror",
"ucd-trie",
]
[[package]]
name = "pest_derive"
version = "2.7.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d3a6e3394ec80feb3b6393c725571754c6188490265c61aaf260810d6b95aa0"
dependencies = [
"pest",
"pest_generator",
]
[[package]]
name = "pest_generator"
version = "2.7.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94429506bde1ca69d1b5601962c73f4172ab4726571a59ea95931218cb0e930e"
dependencies = [
"pest",
"pest_meta",
"proc-macro2",
"quote",
"syn 2.0.77",
]
[[package]]
name = "pest_meta"
version = "2.7.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac8a071862e93690b6e34e9a5fb8e33ff3734473ac0245b27232222c4906a33f"
dependencies = [
"once_cell",
"pest",
"sha2",
]
[[package]]
name = "pin-project"
version = "1.1.5"
@ -2302,6 +2711,31 @@ dependencies = [
"unicase",
]
[[package]]
name = "pulp"
version = "0.18.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6"
dependencies = [
"bytemuck",
"libm",
"num-complex",
"reborrow",
]
[[package]]
name = "py_literal"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "102df7a3d46db9d3891f178dcc826dc270a6746277a9ae6436f8d29fd490a8e1"
dependencies = [
"num-bigint",
"num-complex",
"num-traits",
"pest",
"pest_derive",
]
[[package]]
name = "qoi"
version = "0.4.1"
@ -2437,6 +2871,15 @@ dependencies = [
"rgb",
]
[[package]]
name = "raw-cpuid"
version = "10.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "rawpointer"
version = "0.2.1"
@ -2463,6 +2906,12 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "reborrow"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430"
[[package]]
name = "redox_syscall"
version = "0.5.4"
@ -2868,6 +3317,12 @@ dependencies = [
"serde",
]
[[package]]
name = "seq-macro"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
[[package]]
name = "serde"
version = "1.0.210"
@ -3035,6 +3490,7 @@ dependencies = [
"chrono",
"clap",
"discord-rich-presence",
"faer",
"freetype-rs",
"hypertesseract",
"image 0.25.2",
@ -3200,6 +3656,20 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "sysctl"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea"
dependencies = [
"bitflags 2.6.0",
"byteorder",
"enum-as-inner",
"libc",
"thiserror",
"walkdir",
]
[[package]]
name = "system-configuration"
version = "0.5.1"
@ -3658,6 +4128,12 @@ dependencies = [
"syn 2.0.77",
]
[[package]]
name = "ucd-trie"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
[[package]]
name = "unicase"
version = "2.7.0"

View file

@ -52,6 +52,13 @@ axum = { version = "0.7.6", features = ["macros"] }
paste = "1.0.15"
discord-rich-presence = "0.2.4"
reqwest = { version = "0.12.7", features = ["json"] }
faer = { git = "https://github.com/sarah-ek/faer-rs", rev = "4f3eb7e65c69f7f7df3bdd93aa868d5666db3656", features = ["serde"] }
# [profile.dev.package."*"]
# opt-level = 3
[profile.dev.package.imageproc]
opt-level = 3
[profile.dev.package.image]
opt-level = 3
[profile.dev.package.faer]
opt-level = 3

View file

@ -77,7 +77,9 @@ Additionally, you must place a custom `b30` background at `$SHIMMERING_ASSET_DIR
After everything has been placed in the right directory, run `shimmeringmoon-cli prepare-jackets` to prepare everything. This will:
- Associate each asset with it's database ID
- Build out a recognition matrix for image recognition purposes (this matrix more or less contains a 8x8 downscaled version of each provided asset, stored in bitmap format together with the associated database ID)
- Build out a recognition matrix (~30kb) for image recognition purposes. This file contains:
- about ~3 pixels worth of information for each jacket, stored together with their associated database IDs
- a projection matrix which transforms a $8 \times 8$ downscaled vectorized version of an image (that's $192$ dimensions — $64 \text{pixels} \times 3 \text{channels}$) and projects it to a $10$-dimensional space (the matrix is built using [singular value decomposition](https://en.wikipedia.org/wiki/Singular_value_decomposition)).
### Importing charts

View file

@ -21,7 +21,7 @@
# };
# toolchain = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.default);
# toolchain = pkgs.rust-bin.stable.latest.default;
# toolchain = inputs.fenix.packages.${system}.complete.toolchain;
toolchain = inputs.fenix.packages.${system}.complete.toolchain;
spkgs = inputs.self.packages.${system};
inherit (pkgs) lib;
in
@ -46,11 +46,12 @@
# {{{ Devshell
devShell = pkgs.mkShell rec {
nativeBuildInputs = with pkgs; [
cargo
rustc
clippy
rust-analyzer
rustfmt
# cargo
# rustc
# clippy
# rust-analyzer
# rustfmt
toolchain
ruff
imagemagick

View file

@ -11,6 +11,7 @@ Kanbu de Tomatte Sugu Tokeru,,,overdrive
1F√,,,onefr
[X],,,infinity
0xe0e1ccull,,,ifirmx
Last,,,last
Last | Moment,,,last
Last | Eternity,,,lasteternity
Lost Emotion feat. nomico,,,lostemotion
@ -53,9 +54,11 @@ Einherjar Joker,,,einherjar
GOODTEK (Arcaea Edit),,,goodtek
Kanagawa Cyber Culvert,,,kanagawa
La'qryma of the Wasteland,,,laqryma
PRAGMATISM,,,pragmatism
PRAGMATISM -RESURRECTION-,,,pragmatism
qualia -ideaesthesia-,,,qualia
Shades of Light in a Transcendent Realm,,,shadesoflight
trappola bewitching,,,trappola
Vicious Heroism,,,viciousheroism
Vicious [ANTi] Heroism,,,viciousheroism
eden,,,edenwacca

1 Name Difficulty Artist Shorthand
11 1F√ onefr
12 [X] infinity
13 0xe0e1ccull ifirmx
14 Last last
15 Last | Moment last
16 Last | Eternity lasteternity
17 Lost Emotion feat. nomico lostemotion
54 GOODTEK (Arcaea Edit) goodtek
55 Kanagawa Cyber Culvert kanagawa
56 La'qryma of the Wasteland laqryma
57 PRAGMATISM pragmatism
58 PRAGMATISM -RESURRECTION- pragmatism
59 qualia -ideaesthesia- qualia
60 Shades of Light in a Transcendent Realm shadesoflight
61 trappola bewitching trappola
62 Vicious Heroism viciousheroism
63 Vicious [ANTi] Heroism viciousheroism
64 eden edenwacca

View file

@ -2,13 +2,13 @@
use std::fs;
use anyhow::Context;
use image::{imageops::FilterType, GenericImageView, Rgba};
use num::Integer;
use faer::{Mat, MatRef};
use image::{GenericImageView, Pixel};
use num::{Integer, ToPrimitive};
use serde::{Deserialize, Serialize};
use serde_with::serde_as;
use crate::arcaea::chart::{Difficulty, Jacket, SongCache};
use crate::assets::{get_asset_dir, should_skip_jacket_art};
use crate::assets::get_asset_dir;
use crate::context::Error;
// }}}
@ -16,19 +16,12 @@ use crate::context::Error;
pub const SPLIT_FACTOR: u32 = 8;
pub const IMAGE_VEC_DIM: usize = (SPLIT_FACTOR * SPLIT_FACTOR * 3) as usize;
pub const BITMAP_IMAGE_SIZE: u32 = 174;
pub const JACKET_RECOGNITITION_DIMENSIONS: usize = 10;
#[serde_as]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ImageVec {
#[serde_as(as = "[_; IMAGE_VEC_DIM]")]
pub colors: [f32; IMAGE_VEC_DIM],
}
impl ImageVec {
// {{{ (Image => vector) encoding
#[allow(clippy::identity_op)]
pub fn from_image(image: &impl GenericImageView<Pixel = Rgba<u8>>) -> Self {
let mut colors = [0.0; IMAGE_VEC_DIM];
pub fn image_to_vec(image: &impl GenericImageView) -> MVec<f32> {
let mut colors = MVec::zeros(IMAGE_VEC_DIM, 1);
let chunk_width = image.width() / SPLIT_FACTOR;
let chunk_height = image.height() / SPLIT_FACTOR;
for i in 0..(SPLIT_FACTOR * SPLIT_FACTOR) {
@ -46,9 +39,13 @@ impl ImageVec {
let mut count = 0;
for (_, _, pixel) in cropped.pixels() {
r += (pixel.0[0] as u64).pow(2);
g += (pixel.0[1] as u64).pow(2);
b += (pixel.0[2] as u64).pow(2);
let channels = pixel.channels();
// I'm not sure this does what it's supposed to do for non rgb(a) pixels...
r += channels[0].to_u64().unwrap().pow(2);
g += channels[1].to_u64().unwrap().pow(2);
b += channels[2].to_u64().unwrap().pow(2);
count += 1;
}
@ -56,75 +53,50 @@ impl ImageVec {
let r = (r as f64 / count).sqrt();
let g = (g as f64 / count).sqrt();
let b = (b as f64 / count).sqrt();
colors[i as usize * 3 + 0] = r as f32;
colors[i as usize * 3 + 1] = g as f32;
colors[i as usize * 3 + 2] = b as f32;
colors[(i as usize * 3 + 0, 0)] = r as f32;
colors[(i as usize * 3 + 1, 0)] = g as f32;
colors[(i as usize * 3 + 2, 0)] = b as f32;
}
Self { colors }
}
#[inline]
pub fn distance_squared_to(&self, other: &Self) -> f32 {
let mut total = 0.0;
for i in 0..IMAGE_VEC_DIM {
let d = self.colors[i] - other.colors[i];
total += d * d;
}
total
colors
}
// }}}
}
#[derive(Clone)]
/// A column vector
pub type MVec<T> = Mat<T>;
/// This struct holds:
/// - a set of (song_id, vec) pairs of different images projected through the
/// aforementioned transform.
/// - an projection matrix for dimensionality reduction
#[derive(Clone, Serialize, Deserialize)]
pub struct JacketCache {
jackets: Vec<(u32, ImageVec)>,
/// A matrix with each column corresponding to the result of passing a jacket
/// through [[image_to_vec]], and then projecting it through `transform_matrix`
pub jacket_matrix: Mat<f32>,
/// Assigns each column of `jacket_matrix` a song id.
pub jacket_ids: Vec<u32>,
/// A projection matrix for dimensionality reduction.
pub transform_matrix: Mat<f32>,
}
impl JacketCache {
// {{{ Generate
// This is a bit inefficient (using a hash set), but only runs once
pub fn new(song_cache: &mut SongCache) -> Result<Self, Error> {
let jacket_vectors = if should_skip_jacket_art() {
let path = get_asset_dir().join("placeholder_jacket.jpg");
let contents: &'static _ = fs::read(path)?.leak();
let image = image::load_from_memory(contents)?;
let bitmap: &'static _ = Box::leak(Box::new(
image
.resize(BITMAP_IMAGE_SIZE, BITMAP_IMAGE_SIZE, FilterType::Nearest)
.into_rgb8(),
));
for chart in song_cache.charts_mut() {
chart.cached_jacket = Some(Jacket {
raw: contents,
bitmap,
});
}
Vec::new()
} else {
// {{{ Read jackets
pub fn read_jackets(song_cache: &mut SongCache) -> Result<(), Error> {
let suffix = format!("_{BITMAP_IMAGE_SIZE}.jpg");
let songs_dir = get_asset_dir().join("songs/by_id");
let entries =
fs::read_dir(songs_dir).with_context(|| "Couldn't read songs directory")?;
let bytes = fs::read(get_asset_dir().join("songs/recognition_matrix"))
.with_context(|| "Could not read jacket recognition matrix")?;
let jacket_vectors = postcard::from_bytes(&bytes)
.with_context(|| "Could not decode jacket recognition matrix")?;
let entries = fs::read_dir(songs_dir).with_context(|| "Couldn't read songs directory")?;
for entry in entries {
let dir = entry?;
let raw_dir_name = dir.file_name();
let dir_name = raw_dir_name.to_str().unwrap();
let song_id = dir_name.parse().with_context(|| {
format!("Dir name {dir_name} could not be parsed as `u32` song id")
})?;
let song_id = dir_name
.parse()
.with_context(|| format!("Dir name {dir_name} could not be parsed as `u32` song id"))?;
let entries =
fs::read_dir(dir.path()).with_context(|| "Couldn't read song directory")?;
let entries = fs::read_dir(dir.path()).with_context(|| "Couldn't read song directory")?;
for entry in entries {
let file = entry?;
let raw_name = file.file_name();
@ -172,28 +144,42 @@ impl JacketCache {
}
}
jacket_vectors
};
Ok(())
}
// }}}
let result = Self {
jackets: jacket_vectors,
};
impl JacketCache {
// {{{ Generate
pub fn new() -> Result<Self, Error> {
let bytes = fs::read(get_asset_dir().join("songs/recognition_matrix"))
.with_context(|| "Could not read jacket recognition matrix")?;
let result = postcard::from_bytes(&bytes)?;
// .with_context(|| "Could not decode jacket recognition matrix")?;
Ok(result)
}
// }}}
// {{{ Recognise
/// Transforms a vector from image space to recognition space.
#[inline]
pub fn recognise(
&self,
image: &impl GenericImageView<Pixel = Rgba<u8>>,
) -> Option<(f32, &u32)> {
let vec = ImageVec::from_image(image);
self.jackets
pub fn transform_vec(&self, vec: MatRef<f32>) -> MVec<f32> {
&self.transform_matrix * vec
}
#[inline]
pub fn recognise(&self, image: &impl GenericImageView) -> Option<(f32, u32)> {
let vec = self.transform_vec(image_to_vec(image).as_ref());
self.jacket_ids
.iter()
.map(|(i, v)| (i, v, v.distance_squared_to(&vec)))
.min_by(|(_, _, d1), (_, _, d2)| d1.partial_cmp(d2).expect("NaN distance encountered"))
.map(|(i, _, d)| (d.sqrt(), i))
.enumerate()
.map(|(idx, id)| {
(id, {
(self.jacket_matrix.subcols(idx, 1) - &vec).squared_norm_l2()
})
})
.min_by(|(_, d1), (_, d2)| d1.partial_cmp(d2).expect("NaN distance encountered"))
.map(|(i, d)| (d.sqrt(), *i))
}
// }}}
}

View file

@ -71,11 +71,6 @@ pub static UNI_FONT: RefCell<Face> = get_font("unifont.otf");
}
// }}}
// {{{ Asset art helpers
#[inline]
pub fn should_skip_jacket_art() -> bool {
var("SHIMMERING_NO_JACKETS").unwrap_or_default() == "1"
}
#[inline]
#[allow(dead_code)]
pub fn should_blur_jacket_art() -> bool {

View file

@ -3,10 +3,14 @@ use std::fs;
use std::io::{stdout, Write};
use anyhow::{anyhow, bail, Context};
use faer::Mat;
use image::imageops::FilterType;
use shimmeringmoon::arcaea::chart::{Difficulty, SongCache};
use shimmeringmoon::arcaea::jacket::{ImageVec, BITMAP_IMAGE_SIZE};
use shimmeringmoon::arcaea::jacket::{
image_to_vec, read_jackets, JacketCache, BITMAP_IMAGE_SIZE, IMAGE_VEC_DIM,
JACKET_RECOGNITITION_DIMENSIONS,
};
use shimmeringmoon::assets::{get_asset_dir, get_data_dir};
use shimmeringmoon::context::{connect_db, Error};
use shimmeringmoon::recognition::fuzzy_song_name::guess_chart_name;
@ -20,8 +24,12 @@ fn clear_line() {
pub fn run() -> Result<(), Error> {
let db = connect_db(&get_data_dir());
let song_cache = SongCache::new(&db)?;
let mut song_cache = SongCache::new(&db)?;
let mut jacket_vector_ids = vec![];
let mut jacket_vectors = vec![];
// {{{ Prepare directories
let songs_dir = get_asset_dir().join("songs");
let raw_songs_dir = songs_dir.join("raw");
@ -30,9 +38,8 @@ pub fn run() -> Result<(), Error> {
fs::remove_dir_all(&by_id_dir).with_context(|| "Could not remove `by_id` dir")?;
}
fs::create_dir_all(&by_id_dir).with_context(|| "Could not create `by_id` dir")?;
let mut jacket_vectors = vec![];
// }}}
// {{{ Traverse raw songs directory
let entries = fs::read_dir(&raw_songs_dir)
.with_context(|| "Couldn't read songs directory")?
.collect::<Result<Vec<_>, _>>()
@ -84,12 +91,7 @@ pub fn run() -> Result<(), Error> {
// the same directory. To do this, we only allow the base jacket to refer
// to the FUTURE difficulty, unless it's the only jacket present
// (or unless we are parsing the tutorial)
let search_difficulty =
if entries.len() > 1 && difficulty.is_none() && dir_name != "tutorial" {
Some(Difficulty::FTR)
} else {
difficulty
};
let search_difficulty = difficulty;
let (song, _) = guess_chart_name(dir_name, &song_cache, search_difficulty, true)
.with_context(|| format!("Could not recognise chart name from '{dir_name}'"))?;
@ -120,12 +122,12 @@ pub fn run() -> Result<(), Error> {
.with_context(|| format!("Could not read image for file {:?}", file.path()))?
.leak();
let image = image::load_from_memory(contents)?;
jacket_vectors.push((song.id, ImageVec::from_image(&image)));
let small_image =
image.resize(BITMAP_IMAGE_SIZE, BITMAP_IMAGE_SIZE, FilterType::Gaussian);
jacket_vector_ids.push(song.id);
jacket_vectors.push(image_to_vec(&image));
{
let image_small_path =
out_dir.join(format!("{difficulty_string}_{BITMAP_IMAGE_SIZE}.jpg"));
@ -150,27 +152,100 @@ pub fn run() -> Result<(), Error> {
}
}
}
// }}}
clear_line();
println!("Successfully processed jackets");
// NOTE: this is N^2, but it's a one-off warning thing, so it's fine
read_jackets(&mut song_cache)?;
println!("Successfully read jackets");
// {{{ Warn on missing jackets
for chart in song_cache.charts() {
if jacket_vectors.iter().all(|(i, _)| chart.song_id != *i) {
if chart.cached_jacket.is_none() {
println!(
"No jacket found for '{} [{:?}]'",
song_cache.lookup_song(chart.song_id)?.song.title,
song_cache.lookup_song(chart.song_id)?.song,
chart.difficulty
)
}
}
println!("No missing jackets detected");
// }}}
// {{{ Compute jacket vec matrix
let mut jacket_matrix: Mat<f32> = Mat::zeros(IMAGE_VEC_DIM, jacket_vectors.len());
for (i, v) in jacket_vectors.iter().enumerate() {
jacket_matrix.subcols_mut(i, 1).copy_from(v);
}
// }}}
// {{{ Compute transform matrix
let transform_matrix = {
let svd = jacket_matrix.thin_svd();
svd.u()
.transpose()
.submatrix(0, 0, JACKET_RECOGNITITION_DIMENSIONS, IMAGE_VEC_DIM)
.to_owned()
};
// }}}
// {{{ Build jacket cache
let jacket_cache = JacketCache {
jacket_ids: jacket_vector_ids,
jacket_matrix: &transform_matrix * &jacket_matrix,
transform_matrix,
};
// }}}
// {{{ Perform jacket recognition test
let chart_count = song_cache.charts().count();
for (i, chart) in song_cache.charts().enumerate() {
let song = &song_cache.lookup_song(chart.song_id)?.song;
// {{{ Update console display
if i != 0 {
clear_line();
}
print!("{}/{}: {song}", i, chart_count);
if i % 5 == 0 {
stdout().flush()?;
}
// }}}
if let Some(jacket) = chart.cached_jacket {
if let Some((_, song_id)) = jacket_cache.recognise(jacket.bitmap) {
if song_id != song.id {
let mistake = &song_cache.lookup_song(song_id)?.song;
bail!(
"Could not recognise jacket for {song} [{}]. Found song {mistake} instead.",
chart.difficulty
)
}
} else {
bail!(
"Could not recognise jacket for {song} [{}].",
chart.difficulty
)
}
}
}
// }}}
clear_line();
println!("Successfully tested jacket recognition");
// {{{ Save recognition matrix to disk
{
println!("Encoded {} images", jacket_vectors.len());
let bytes = postcard::to_allocvec(&jacket_vectors)
let bytes = postcard::to_allocvec(&jacket_cache)
.with_context(|| "Coult not encode jacket matrix")?;
fs::write(songs_dir.join("recognition_matrix"), bytes)
.with_context(|| "Could not write jacket matrix")?;
}
// }}}
Ok(())
}

View file

@ -7,6 +7,7 @@ use std::fs;
use std::path::Path;
use std::sync::LazyLock;
use crate::arcaea::jacket::read_jackets;
use crate::arcaea::{chart::SongCache, jacket::JacketCache};
use crate::assets::{get_data_dir, EXO_FONT, GEOSANS_FONT, KAZESAWA_BOLD_FONT, KAZESAWA_FONT};
use crate::recognition::{hyperglass::CharMeasurements, ui::UIMeasurements};
@ -109,7 +110,10 @@ impl UserContext {
let mut song_cache = SongCache::new(&db)?;
let ui_measurements = UIMeasurements::read()?;
let jacket_cache = timed!("make_jacket_cache", { JacketCache::new(&mut song_cache)? });
let jacket_cache = JacketCache::new()?;
timed!("read_jackets", {
read_jackets(&mut song_cache)?;
});
// {{{ Font measurements
static WHITELIST: &str = "0123456789'abcdefghklmnopqrstuvwxyzABCDEFGHIJKLMNOPRSTUVWXYZ";

View file

@ -327,7 +327,7 @@ impl ImageAnalyzer {
bail!("No known jacket looks like this");
}
let (song, chart) = ctx.song_cache.lookup_by_difficulty(*song_id, difficulty)?;
let (song, chart) = ctx.song_cache.lookup_by_difficulty(song_id, difficulty)?;
Ok((song, chart))
}