No longer use tesseract for score OCR (tesseract is terrrrrible)
Signed-off-by: prescientmoon <git@moonythm.dev>
This commit is contained in:
parent
86e5debe95
commit
4373b6ca62
SSH key fingerprint:
SHA256:UUF9JT2s8Xfyv76b8ZuVL7XrmimH4o49p4b+iexbVH4
142
Cargo.lock
generated
142
Cargo.lock
generated
|
|
@ -2,6 +2,22 @@
|
|||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
|
||||
[[package]]
|
||||
name = "ab_glyph"
|
||||
version = "0.2.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79faae4620f45232f599d9bc7b290f88247a0834162c4495ab2f02d60004adfb"
|
||||
dependencies = [
|
||||
"ab_glyph_rasterizer",
|
||||
"owned_ttf_parser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ab_glyph_rasterizer"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c71b1793ee61086797f5c80b6efa2b8ffa6d5dd703f118545808a7f2e27f7046"
|
||||
|
||||
[[package]]
|
||||
name = "addr2line"
|
||||
version = "0.22.0"
|
||||
|
|
@ -71,6 +87,15 @@ version = "1.0.86"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
|
||||
|
||||
[[package]]
|
||||
name = "approx"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arbitrary"
|
||||
version = "1.3.2"
|
||||
|
|
@ -989,8 +1014,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"js-sys",
|
||||
"libc",
|
||||
"wasi",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1204,7 +1231,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "hypertesseract"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/BlueGhostGH/hypertesseract.git?rev=78dd8ab#78dd8ab1bbab9d7985959a5a8ac2746bce17ff5c"
|
||||
source = "git+https://github.com/BlueGhostGH/hypertesseract.git?rev=4e05063#4e050634f50a58b9be85018439a0b1a23b59de35"
|
||||
dependencies = [
|
||||
"image 0.25.2",
|
||||
"sys",
|
||||
|
|
@ -1297,6 +1324,24 @@ dependencies = [
|
|||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "imageproc"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2393fb7808960751a52e8a154f67e7dd3f8a2ef9bd80d1553078a7b4e8ed3f0d"
|
||||
dependencies = [
|
||||
"ab_glyph",
|
||||
"approx",
|
||||
"getrandom",
|
||||
"image 0.25.2",
|
||||
"itertools",
|
||||
"nalgebra",
|
||||
"num",
|
||||
"rand",
|
||||
"rand_distr",
|
||||
"rayon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "imgref"
|
||||
version = "1.10.1"
|
||||
|
|
@ -1469,6 +1514,16 @@ dependencies = [
|
|||
"imgref",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matrixmultiply"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"rawpointer",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "maybe-rayon"
|
||||
version = "0.1.1"
|
||||
|
|
@ -1553,6 +1608,21 @@ dependencies = [
|
|||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nalgebra"
|
||||
version = "0.32.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b5c17de023a86f59ed79891b2e5d5a94c705dbe904a5b5c9c952ea6221b03e4"
|
||||
dependencies = [
|
||||
"approx",
|
||||
"matrixmultiply",
|
||||
"num-complex",
|
||||
"num-rational",
|
||||
"num-traits",
|
||||
"simba",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "new_debug_unreachable"
|
||||
version = "1.0.6"
|
||||
|
|
@ -1720,6 +1790,15 @@ dependencies = [
|
|||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "owned_ttf_parser"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "490d3a563d3122bf7c911a59b0add9389e5ec0f5f0c3ac6b91ff235a0e6a7f90"
|
||||
dependencies = [
|
||||
"ttf-parser 0.24.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking"
|
||||
version = "2.2.0"
|
||||
|
|
@ -1842,7 +1921,7 @@ dependencies = [
|
|||
"plotters-backend",
|
||||
"plotters-bitmap",
|
||||
"plotters-svg",
|
||||
"ttf-parser",
|
||||
"ttf-parser 0.15.2",
|
||||
"wasm-bindgen",
|
||||
"web-sys",
|
||||
]
|
||||
|
|
@ -2017,6 +2096,16 @@ dependencies = [
|
|||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_distr"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
"rand",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rav1e"
|
||||
version = "0.7.1"
|
||||
|
|
@ -2067,6 +2156,12 @@ dependencies = [
|
|||
"rgb",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rawpointer"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.10.0"
|
||||
|
|
@ -2329,6 +2424,15 @@ version = "1.0.18"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
|
||||
|
||||
[[package]]
|
||||
name = "safe_arch"
|
||||
version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3460605018fdc9612bce72735cba0d27efbcd9904780d44c7e3a9948f96148a"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "same-file"
|
||||
version = "1.0.6"
|
||||
|
|
@ -2497,6 +2601,7 @@ dependencies = [
|
|||
"freetype-rs",
|
||||
"hypertesseract",
|
||||
"image 0.25.2",
|
||||
"imageproc",
|
||||
"num",
|
||||
"plotters",
|
||||
"poise",
|
||||
|
|
@ -2514,6 +2619,19 @@ dependencies = [
|
|||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simba"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "061507c94fc6ab4ba1c9a0305018408e312e17c041eb63bef8aa726fa33aceae"
|
||||
dependencies = [
|
||||
"approx",
|
||||
"num-complex",
|
||||
"num-traits",
|
||||
"paste",
|
||||
"wide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.7"
|
||||
|
|
@ -2859,7 +2977,7 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
|
|||
[[package]]
|
||||
name = "sys"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/BlueGhostGH/hypertesseract.git?rev=78dd8ab#78dd8ab1bbab9d7985959a5a8ac2746bce17ff5c"
|
||||
source = "git+https://github.com/BlueGhostGH/hypertesseract.git?rev=4e05063#4e050634f50a58b9be85018439a0b1a23b59de35"
|
||||
dependencies = [
|
||||
"openssl-sys",
|
||||
"pkg-config",
|
||||
|
|
@ -2927,7 +3045,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "thin"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/BlueGhostGH/hypertesseract.git?rev=78dd8ab#78dd8ab1bbab9d7985959a5a8ac2746bce17ff5c"
|
||||
source = "git+https://github.com/BlueGhostGH/hypertesseract.git?rev=4e05063#4e050634f50a58b9be85018439a0b1a23b59de35"
|
||||
dependencies = [
|
||||
"sys",
|
||||
]
|
||||
|
|
@ -3188,6 +3306,12 @@ version = "0.15.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b3e06c9b9d80ed6b745c7159c40b311ad2916abb34a49e9be2653b90db0d8dd"
|
||||
|
||||
[[package]]
|
||||
name = "ttf-parser"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5be21190ff5d38e8b4a2d3b6a3ae57f612cc39c96e83cedeaf7abc338a8bac4a"
|
||||
|
||||
[[package]]
|
||||
name = "tungstenite"
|
||||
version = "0.21.0"
|
||||
|
|
@ -3496,6 +3620,16 @@ dependencies = [
|
|||
"wasite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wide"
|
||||
version = "0.7.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "901e8597c777fa042e9e245bd56c0dc4418c5db3f845b6ff94fbac732c6a0692"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
"safe_arch",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
|
|
|
|||
|
|
@ -11,8 +11,9 @@ num = "0.4.3"
|
|||
plotters = { git="https://github.com/starlitcanopy/plotters.git", rev="986cd959362a2dbec8d1b25670fd083b904d7b8c", features=["bitmap_backend"] }
|
||||
poise = "0.6.1"
|
||||
sqlx = { version = "0.8.0", features = ["sqlite", "runtime-tokio", "chrono"] }
|
||||
hypertesseract = { features=["image"], git="https://github.com/BlueGhostGH/hypertesseract.git", rev="78dd8ab" }
|
||||
hypertesseract = { features=["image"], git="https://github.com/BlueGhostGH/hypertesseract.git", rev="4e05063" }
|
||||
tokio = {version="1.38.0", features=["rt-multi-thread"]}
|
||||
imageproc = "0.25.0"
|
||||
|
||||
[profile.dev.package."*"]
|
||||
opt-level = 3
|
||||
|
|
|
|||
863
data/charts.csv
863
data/charts.csv
File diff suppressed because it is too large
Load diff
22
scripts/import_jacket.sh
Executable file
22
scripts/import_jacket.sh
Executable file
|
|
@ -0,0 +1,22 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
if [ "$#" != 2 ]; then
|
||||
echo "Usage: $0 <name> <url>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
name=$1
|
||||
url=$2
|
||||
|
||||
curr=$(pwd)
|
||||
|
||||
dir_path=$SHIMMERING_DATA_DIR/songs/$name
|
||||
mkdir $dir_path
|
||||
cd $dir_path
|
||||
|
||||
http GET "$url" > temp
|
||||
convert ./temp ./base.jpg
|
||||
convert ./base.jpg -resize 256x256 ./base_256.jpg
|
||||
rm temp
|
||||
|
||||
cd $curr
|
||||
|
|
@ -52,7 +52,7 @@ def import_charts_from_csv():
|
|||
[note_design, level, cc, note_count] = charts[i * 4 : (i + 1) * 4]
|
||||
if note_design == "N/A":
|
||||
continue
|
||||
chart_count += 2
|
||||
chart_count += 1
|
||||
|
||||
[difficulty, level] = level.split(" ")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#!/usr/bin/env bash
|
||||
dir_path=./data/songs
|
||||
dir_path=$SHIMMERING_DATA_DIR/songs
|
||||
|
||||
# Find all files in the directory and its subdirectories
|
||||
find "$dir_path" -type f | while read -r file; do
|
||||
|
|
|
|||
4
scripts/reimport-songs.sh
Executable file
4
scripts/reimport-songs.sh
Executable file
|
|
@ -0,0 +1,4 @@
|
|||
#!/usr/bin/env bash
|
||||
echo "delete from songs" | sqlite3 $SHIMMERING_DATA_DIR/db.sqlite
|
||||
echo "delete from charts" | sqlite3 $SHIMMERING_DATA_DIR/db.sqlite
|
||||
./scripts/main.py import charts
|
||||
|
|
@ -89,6 +89,8 @@ pub struct Song {
|
|||
pub id: u32,
|
||||
pub title: String,
|
||||
pub lowercase_title: String,
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub artist: String,
|
||||
|
||||
pub bpm: String,
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ pub struct ImageVec {
|
|||
|
||||
impl ImageVec {
|
||||
// {{{ (Image => vector) encoding
|
||||
fn from_image(image: &impl GenericImageView<Pixel = Rgba<u8>>) -> ImageVec {
|
||||
fn from_image(image: &impl GenericImageView<Pixel = Rgba<u8>>) -> Self {
|
||||
let mut colors = [0.0; IMAGE_VEC_DIM];
|
||||
let chunk_width = image.width() / SPLIT_FACTOR;
|
||||
let chunk_height = image.height() / SPLIT_FACTOR;
|
||||
|
|
@ -176,6 +176,16 @@ impl JacketCache {
|
|||
}
|
||||
}
|
||||
|
||||
for chart in song_cache.charts() {
|
||||
if chart.cached_jacket.is_none() {
|
||||
println!(
|
||||
"No jacket found for '{} [{:?}]'",
|
||||
song_cache.lookup_song(chart.song_id)?.song.title,
|
||||
chart.difficulty
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
jacket_vectors
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -170,162 +170,52 @@ impl Score {
|
|||
}
|
||||
// }}}
|
||||
// {{{ Scores & Distribution => score
|
||||
pub fn resolve_ambiguities(
|
||||
scores: Vec<Score>,
|
||||
pub fn resolve_distibution_ambiguities(
|
||||
score: Score,
|
||||
read_distribution: Option<(u32, u32, u32)>,
|
||||
note_count: u32,
|
||||
) -> Result<(Score, Option<u32>, Option<&'static str>), Error> {
|
||||
if scores.len() == 0 {
|
||||
return Err("No scores in list to disambiguate from.")?;
|
||||
}
|
||||
) -> Option<u32> {
|
||||
let read_distribution = read_distribution?;
|
||||
let pures = read_distribution.0;
|
||||
let fars = read_distribution.1;
|
||||
let losts = read_distribution.2;
|
||||
|
||||
let mut no_shiny_scores: Vec<_> = scores
|
||||
.iter()
|
||||
.map(|score| score.forget_shinies(note_count))
|
||||
.collect();
|
||||
no_shiny_scores.sort();
|
||||
no_shiny_scores.dedup();
|
||||
|
||||
if let Some(read_distribution) = read_distribution {
|
||||
let pures = read_distribution.0;
|
||||
let fars = read_distribution.1;
|
||||
let losts = read_distribution.2;
|
||||
|
||||
// Compute score from note breakdown subpairs
|
||||
let pf_score = Score::compute_naive(note_count, pures, fars);
|
||||
let fl_score = Score::compute_naive(
|
||||
note_count,
|
||||
note_count.checked_sub(losts + fars).unwrap_or(0),
|
||||
fars,
|
||||
);
|
||||
let lp_score = Score::compute_naive(
|
||||
note_count,
|
||||
pures,
|
||||
note_count.checked_sub(losts + pures).unwrap_or(0),
|
||||
);
|
||||
|
||||
if no_shiny_scores.len() == 1 {
|
||||
// {{{ Score is fixed, gotta figure out the exact distribution
|
||||
let score = *scores.iter().max().unwrap();
|
||||
|
||||
// {{{ Look for consensus among recomputed scores
|
||||
// Lemma: if two computed scores agree, then so will the third
|
||||
let consensus_fars = if pf_score == fl_score {
|
||||
Some(fars)
|
||||
} else {
|
||||
// Due to the above lemma, we know all three scores must be distinct by
|
||||
// this point.
|
||||
//
|
||||
// Our strategy is to check which of the three scores agrees with the real
|
||||
// score, and to then trust the `far` value that contributed to that pair.
|
||||
let no_shiny_score = score.forget_shinies(note_count);
|
||||
let pf_appears = no_shiny_score == pf_score;
|
||||
let fl_appears = no_shiny_score == fl_score;
|
||||
let lp_appears = no_shiny_score == lp_score;
|
||||
|
||||
match (pf_appears, fl_appears, lp_appears) {
|
||||
(true, false, false) => Some(fars),
|
||||
(false, true, false) => Some(fars),
|
||||
(false, false, true) => Some(note_count - pures - losts),
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
// }}}
|
||||
|
||||
if scores.len() == 1 {
|
||||
Ok((score, consensus_fars, None))
|
||||
} else {
|
||||
Ok((score, consensus_fars, Some("Due to a reading error, I could not make sure the shiny-amount I calculated is accurate!")))
|
||||
}
|
||||
|
||||
// }}}
|
||||
} else {
|
||||
// {{{ Score is not fixed, gotta figure out everything at once
|
||||
// Some of the values in the note distribution are likely wrong (due to reading
|
||||
// errors). To get around this, we take each pair from the triplet, compute the score
|
||||
// it induces, and figure out if there's any consensus as to which value in the
|
||||
// provided score list is the real one.
|
||||
//
|
||||
// Note that sometimes the note distribution cannot resolve any of the issues. This is
|
||||
// usually the case when the disagreement comes from the number of shinies.
|
||||
|
||||
// {{{ Look for consensus among recomputed scores
|
||||
// Lemma: if two computed scores agree, then so will the third
|
||||
let (trusted_pure_count, consensus_computed_score, consensus_fars) = if pf_score
|
||||
== fl_score
|
||||
{
|
||||
(true, pf_score, fars)
|
||||
} else {
|
||||
// Due to the above lemma, we know all three scores must be distinct by
|
||||
// this point.
|
||||
//
|
||||
// Our strategy is to check which of the three scores appear in the
|
||||
// provided score list.
|
||||
let pf_appears = no_shiny_scores.contains(&pf_score);
|
||||
let fl_appears = no_shiny_scores.contains(&fl_score);
|
||||
let lp_appears = no_shiny_scores.contains(&lp_score);
|
||||
|
||||
match (pf_appears, fl_appears, lp_appears) {
|
||||
(true, false, false) => (true, pf_score, fars),
|
||||
(false, true, false) => (false, fl_score, fars),
|
||||
(false, false, true) => (true, lp_score, note_count - pures - losts),
|
||||
_ => Err(format!("Cannot disambiguate scores {:?}. Multiple disjoint note breakdown subpair scores appear on the possibility list", scores))?
|
||||
}
|
||||
};
|
||||
// }}}
|
||||
// {{{ Collect all scores that agree with the consensus score.
|
||||
let agreement: Vec<_> = scores
|
||||
.iter()
|
||||
.filter(|score| score.forget_shinies(note_count) == consensus_computed_score)
|
||||
.filter(|score| {
|
||||
let shinies = score.shinies(note_count);
|
||||
shinies <= note_count && (!trusted_pure_count || shinies <= pures)
|
||||
})
|
||||
.map(|v| *v)
|
||||
.collect();
|
||||
// }}}
|
||||
// {{{ Case 1: Disagreement in the amount of shinies!
|
||||
if agreement.len() > 1 {
|
||||
let agreement_shiny_amounts: Vec<_> =
|
||||
agreement.iter().map(|v| v.shinies(note_count)).collect();
|
||||
|
||||
println!(
|
||||
"Shiny count disagreement. Possible scores: {:?}. Possible shiny amounts: {:?}, Read distribution: {:?}",
|
||||
scores, agreement_shiny_amounts, read_distribution
|
||||
);
|
||||
|
||||
let msg = Some(
|
||||
"Due to a reading error, I could not make sure the shiny-amount I calculated is accurate!"
|
||||
);
|
||||
|
||||
Ok((
|
||||
agreement.into_iter().max().unwrap(),
|
||||
Some(consensus_fars),
|
||||
msg,
|
||||
))
|
||||
// }}}
|
||||
// {{{ Case 2: Total agreement!
|
||||
} else if agreement.len() == 1 {
|
||||
Ok((agreement[0], Some(consensus_fars), None))
|
||||
// }}}
|
||||
// {{{ Case 3: No agreement!
|
||||
} else {
|
||||
Err(format!("Could not disambiguate between possible scores {:?}. Note distribution does not agree with any possibility, leading to a score of {:?}.", scores, consensus_computed_score))?
|
||||
}
|
||||
// }}}
|
||||
// }}}
|
||||
}
|
||||
// {{{ Compute score from note breakdown subpairs
|
||||
let pf_score = Score::compute_naive(note_count, pures, fars);
|
||||
let fl_score = Score::compute_naive(
|
||||
note_count,
|
||||
note_count.checked_sub(losts + fars).unwrap_or(0),
|
||||
fars,
|
||||
);
|
||||
let lp_score = Score::compute_naive(
|
||||
note_count,
|
||||
pures,
|
||||
note_count.checked_sub(losts + pures).unwrap_or(0),
|
||||
);
|
||||
// }}}
|
||||
// {{{ Look for consensus among recomputed scores
|
||||
// Lemma: if two computed scores agree, then so will the third
|
||||
if pf_score == fl_score {
|
||||
Some(fars)
|
||||
} else {
|
||||
if no_shiny_scores.len() == 1 {
|
||||
if scores.len() == 1 {
|
||||
Ok((scores[0], None, None))
|
||||
} else {
|
||||
Ok((scores.into_iter().max().unwrap(), None, Some("Due to a reading error, I could not make sure the shiny-amount I calculated is accurate!")))
|
||||
}
|
||||
} else {
|
||||
Err("Cannot disambiguate between more than one score without a note distribution.")?
|
||||
// Due to the above lemma, we know all three scores must be distinct by
|
||||
// this point.
|
||||
//
|
||||
// Our strategy is to check which of the three scores agrees with the real
|
||||
// score, and to then trust the `far` value that contributed to that pair.
|
||||
let no_shiny_score = score.forget_shinies(note_count);
|
||||
let pf_appears = no_shiny_score == pf_score;
|
||||
let fl_appears = no_shiny_score == fl_score;
|
||||
let lp_appears = no_shiny_score == lp_score;
|
||||
|
||||
match (pf_appears, fl_appears, lp_appears) {
|
||||
(true, false, false) => Some(fars),
|
||||
(false, true, false) => Some(fars),
|
||||
(false, false, true) => Some(note_count - pures - losts),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
// }}}
|
||||
}
|
||||
// }}}
|
||||
// {{{ Display self with diff
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ pub fn get_assets_dir() -> PathBuf {
|
|||
#[inline]
|
||||
fn get_font(name: &str) -> RefCell<Face> {
|
||||
let face = FREETYPE_LIB.with(|lib| {
|
||||
lib.new_face(get_assets_dir().join(format!("{}-variable.ttf", name)), 0)
|
||||
lib.new_face(get_assets_dir().join(format!("{}.ttf", name)), 0)
|
||||
.expect(&format!("Could not load {} font", name))
|
||||
});
|
||||
RefCell::new(face)
|
||||
|
|
@ -27,8 +27,9 @@ fn get_font(name: &str) -> RefCell<Face> {
|
|||
|
||||
thread_local! {
|
||||
pub static FREETYPE_LIB: Library = Library::init().unwrap();
|
||||
pub static SAIRA_FONT: RefCell<Face> = get_font("saira");
|
||||
pub static EXO_FONT: RefCell<Face> = get_font("exo");
|
||||
pub static SAIRA_FONT: RefCell<Face> = get_font("saira-variable");
|
||||
pub static EXO_FONT: RefCell<Face> = get_font("exo-variable");
|
||||
pub static GEOSANS_FONT: RefCell<Face> = get_font("geosans-light");
|
||||
}
|
||||
|
||||
#[inline]
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ fn float_to_ft_fixed(f: f32) -> i64 {
|
|||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct TextStyle {
|
||||
pub size: u32,
|
||||
pub weight: u32,
|
||||
pub weight: Option<u32>,
|
||||
pub color: Color,
|
||||
pub align: (Align, Align),
|
||||
pub stroke: Option<(Color, f32)>,
|
||||
|
|
@ -154,6 +154,11 @@ pub struct BitmapCanvas {
|
|||
}
|
||||
|
||||
impl BitmapCanvas {
|
||||
#[inline]
|
||||
pub fn height(&self) -> u32 {
|
||||
self.buffer.len() as u32 / 3 / self.width
|
||||
}
|
||||
|
||||
// {{{ Draw pixel
|
||||
pub fn set_pixel(&mut self, pos: (u32, u32), color: Color) {
|
||||
let index = 3 * (pos.1 * self.width + pos.0) as usize;
|
||||
|
|
@ -169,7 +174,7 @@ impl BitmapCanvas {
|
|||
// {{{ Draw RBG image
|
||||
/// Draws a bitmap image
|
||||
pub fn blit_rbg(&mut self, pos: Position, (iw, ih): (u32, u32), src: &[u8]) {
|
||||
let height = self.buffer.len() as u32 / 3 / self.width;
|
||||
let height = self.height();
|
||||
for dx in 0..iw {
|
||||
for dy in 0..ih {
|
||||
let x = pos.0 + dx as i32;
|
||||
|
|
@ -190,7 +195,7 @@ impl BitmapCanvas {
|
|||
// {{{ Draw RGBA image
|
||||
/// Draws a bitmap image taking care of the alpha channel.
|
||||
pub fn blit_rbga(&mut self, pos: Position, (iw, ih): (u32, u32), src: &[u8]) {
|
||||
let height = self.buffer.len() as u32 / 3 / self.width;
|
||||
let height = self.height();
|
||||
for dx in 0..iw {
|
||||
for dy in 0..ih {
|
||||
let x = pos.0 + dx as i32;
|
||||
|
|
@ -212,7 +217,7 @@ impl BitmapCanvas {
|
|||
// {{{ Fill
|
||||
/// Fill with solid color
|
||||
pub fn fill(&mut self, pos: Position, (iw, ih): (u32, u32), color: Color) {
|
||||
let height = self.buffer.len() as u32 / 3 / self.width;
|
||||
let height = self.height();
|
||||
for dx in 0..iw {
|
||||
for dy in 0..ih {
|
||||
let x = pos.0 + dx as i32;
|
||||
|
|
@ -233,23 +238,25 @@ impl BitmapCanvas {
|
|||
text: &str,
|
||||
) -> Result<(Position, Rect, Vec<(i64, Glyph)>), Error> {
|
||||
// {{{ Control weight
|
||||
unsafe {
|
||||
let raw = face.raw_mut() as *mut _;
|
||||
let slice = [(style.weight as i64) << 16];
|
||||
if let Some(weight) = style.weight {
|
||||
unsafe {
|
||||
let raw = face.raw_mut() as *mut _;
|
||||
let slice = [(weight as i64) << 16];
|
||||
|
||||
// {{{ Debug logging
|
||||
// let mut amaster = 0 as *mut FT_MM_Var;
|
||||
// FT_Get_MM_Var(raw, &mut amaster as *mut _);
|
||||
// println!("{:?}", *amaster);
|
||||
// println!("{:?}", *(*amaster).axis);
|
||||
// println!("{:?}", *(*amaster).namedstyle);
|
||||
// }}}
|
||||
// {{{ Debug logging
|
||||
// let mut amaster = 0 as *mut FT_MM_Var;
|
||||
// FT_Get_MM_Var(raw, &mut amaster as *mut _);
|
||||
// println!("{:?}", *amaster);
|
||||
// println!("{:?}", *(*amaster).axis);
|
||||
// println!("{:?}", *(*amaster).namedstyle);
|
||||
// }}}
|
||||
|
||||
// Set variable weight
|
||||
let err = FT_Set_Var_Design_Coordinates(raw, 3, slice.as_ptr());
|
||||
if err != FT_Err_Ok {
|
||||
let err: FtResult<_> = Err(err.into());
|
||||
err?;
|
||||
// Set variable weight
|
||||
let err = FT_Set_Var_Design_Coordinates(raw, 3, slice.as_ptr());
|
||||
if err != FT_Err_Ok {
|
||||
let err: FtResult<_> = Err(err.into());
|
||||
err?;
|
||||
}
|
||||
}
|
||||
}
|
||||
// }}}
|
||||
|
|
@ -418,7 +425,7 @@ impl BitmapCanvas {
|
|||
|
||||
#[inline]
|
||||
pub fn new(width: u32, height: u32) -> Self {
|
||||
let buffer = vec![u8::MAX; 8 * 3 * (width * height) as usize].into_boxed_slice();
|
||||
let buffer = vec![u8::MAX; 3 * (width * height) as usize].into_boxed_slice();
|
||||
Self { buffer, width }
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ pub async fn magic(
|
|||
};
|
||||
|
||||
edit_reply!(ctx, handle, "Image {}: reading score", i + 1).await?;
|
||||
let score_possibilities = analyzer.read_score(
|
||||
let score = analyzer.read_score(
|
||||
ctx.data(),
|
||||
Some(chart.note_count),
|
||||
&ocr_image,
|
||||
|
|
@ -89,17 +89,11 @@ pub async fn magic(
|
|||
)?;
|
||||
|
||||
// {{{ Build play
|
||||
let (score, maybe_fars, score_warning) = Score::resolve_ambiguities(
|
||||
score_possibilities,
|
||||
let maybe_fars = Score::resolve_distibution_ambiguities(
|
||||
score,
|
||||
note_distribution,
|
||||
chart.note_count,
|
||||
)
|
||||
.map_err(|err| {
|
||||
format!(
|
||||
"Error occurred when disambiguating scores for '{}' [{:?}] by {}: {}",
|
||||
song.title, difficulty, song.artist, err
|
||||
)
|
||||
})?;
|
||||
);
|
||||
|
||||
let play = CreatePlay::new(score, &chart, &user)
|
||||
.with_attachment(file)
|
||||
|
|
@ -110,14 +104,10 @@ pub async fn magic(
|
|||
// }}}
|
||||
// }}}
|
||||
// {{{ Deliver embed
|
||||
let (mut embed, attachment) = play
|
||||
let (embed, attachment) = play
|
||||
.to_embed(&ctx.data().db, &user, &song, &chart, i, None)
|
||||
.await?;
|
||||
|
||||
if let Some(warning) = score_warning {
|
||||
embed = embed.description(warning);
|
||||
}
|
||||
|
||||
embeds.push(embed);
|
||||
attachments.extend(attachment);
|
||||
// }}}
|
||||
|
|
@ -139,9 +129,11 @@ pub async fn magic(
|
|||
|
||||
handle.delete(ctx).await?;
|
||||
|
||||
ctx.channel_id()
|
||||
.send_files(ctx.http(), attachments, CreateMessage::new().embeds(embeds))
|
||||
.await?;
|
||||
if embeds.len() > 0 {
|
||||
ctx.channel_id()
|
||||
.send_files(ctx.http(), attachments, CreateMessage::new().embeds(embeds))
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
|
|
|||
|
|
@ -307,7 +307,7 @@ pub async fn b30(ctx: Context<'_>) -> Result<(), Error> {
|
|||
font,
|
||||
crate::bitmap::TextStyle {
|
||||
size: 25,
|
||||
weight: 800,
|
||||
weight: Some(800),
|
||||
color: Color::WHITE,
|
||||
align: (Align::Center, Align::Center),
|
||||
stroke: None,
|
||||
|
|
@ -327,7 +327,7 @@ pub async fn b30(ctx: Context<'_>) -> Result<(), Error> {
|
|||
let initial_size = 24;
|
||||
let mut style = crate::bitmap::TextStyle {
|
||||
size: initial_size,
|
||||
weight: 800,
|
||||
weight: Some(800),
|
||||
color: Color::WHITE,
|
||||
align: (Align::Start, Align::Center),
|
||||
stroke: Some((Color::BLACK, 1.5)),
|
||||
|
|
@ -404,7 +404,7 @@ pub async fn b30(ctx: Context<'_>) -> Result<(), Error> {
|
|||
font,
|
||||
crate::bitmap::TextStyle {
|
||||
size: 25,
|
||||
weight: 600,
|
||||
weight: Some(600),
|
||||
color: Color::from_rgb_int(0xffffff),
|
||||
align: (Align::Center, Align::Center),
|
||||
stroke: None,
|
||||
|
|
@ -442,7 +442,7 @@ pub async fn b30(ctx: Context<'_>) -> Result<(), Error> {
|
|||
font,
|
||||
crate::bitmap::TextStyle {
|
||||
size: 23,
|
||||
weight: 800,
|
||||
weight: Some(800),
|
||||
color: Color::WHITE,
|
||||
align: (Align::Start, Align::Center),
|
||||
stroke: Some((Color::BLACK, 1.5)),
|
||||
|
|
@ -490,7 +490,7 @@ pub async fn b30(ctx: Context<'_>) -> Result<(), Error> {
|
|||
font,
|
||||
crate::bitmap::TextStyle {
|
||||
size: if status == 'M' { 30 } else { 36 },
|
||||
weight: if status == 'M' { 800 } else { 500 },
|
||||
weight: Some(if status == 'M' { 800 } else { 500 }),
|
||||
color: Color::WHITE,
|
||||
align: (Align::Center, Align::Center),
|
||||
stroke: None,
|
||||
|
|
@ -526,7 +526,7 @@ pub async fn b30(ctx: Context<'_>) -> Result<(), Error> {
|
|||
font,
|
||||
crate::bitmap::TextStyle {
|
||||
size: 30,
|
||||
weight: 650,
|
||||
weight: Some(650),
|
||||
color: Color::from_rgb_int(0x203C6B),
|
||||
align: (Align::Center, Align::Center),
|
||||
stroke: Some((Color::WHITE, 1.5)),
|
||||
|
|
@ -540,7 +540,7 @@ pub async fn b30(ctx: Context<'_>) -> Result<(), Error> {
|
|||
EXO_FONT.with_borrow_mut(|font| -> Result<(), Error> {
|
||||
let mut style = crate::bitmap::TextStyle {
|
||||
size: 12,
|
||||
weight: 600,
|
||||
weight: Some(600),
|
||||
color: Color::WHITE,
|
||||
align: (Align::Center, Align::Center),
|
||||
stroke: None,
|
||||
|
|
@ -556,7 +556,7 @@ pub async fn b30(ctx: Context<'_>) -> Result<(), Error> {
|
|||
)?;
|
||||
|
||||
style.size = 25;
|
||||
style.weight = 700;
|
||||
style.weight = Some(700);
|
||||
|
||||
drawer.text(
|
||||
top_left_area,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ use std::{fs, path::PathBuf};
|
|||
use sqlx::SqlitePool;
|
||||
|
||||
use crate::{
|
||||
arcaea::chart::SongCache, arcaea::jacket::JacketCache, recognition::ui::UIMeasurements,
|
||||
arcaea::{chart::SongCache, jacket::JacketCache},
|
||||
assets::{EXO_FONT, GEOSANS_FONT},
|
||||
recognition::{hyperglass::CharMeasurements, ui::UIMeasurements},
|
||||
};
|
||||
|
||||
// Types used by all command functions
|
||||
|
|
@ -19,6 +21,9 @@ pub struct UserContext {
|
|||
pub song_cache: SongCache,
|
||||
pub jacket_cache: JacketCache,
|
||||
pub ui_measurements: UIMeasurements,
|
||||
|
||||
pub geosans_measurements: CharMeasurements,
|
||||
pub exo_measurements: CharMeasurements,
|
||||
}
|
||||
|
||||
impl UserContext {
|
||||
|
|
@ -30,6 +35,10 @@ impl UserContext {
|
|||
let mut song_cache = SongCache::new(&db).await?;
|
||||
let jacket_cache = JacketCache::new(&data_dir, &mut song_cache)?;
|
||||
let ui_measurements = UIMeasurements::read(&data_dir)?;
|
||||
let geosans_measurements = GEOSANS_FONT
|
||||
.with_borrow_mut(|font| CharMeasurements::from_text(font, "0123456789'", None))?;
|
||||
let exo_measurements = EXO_FONT
|
||||
.with_borrow_mut(|font| CharMeasurements::from_text(font, "0123456789'", Some(700)))?;
|
||||
|
||||
println!("Created user context");
|
||||
|
||||
|
|
@ -39,6 +48,8 @@ impl UserContext {
|
|||
song_cache,
|
||||
jacket_cache,
|
||||
ui_measurements,
|
||||
geosans_measurements,
|
||||
exo_measurements,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
36
src/logs.rs
Normal file
36
src/logs.rs
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
use std::{env, ops::Deref};
|
||||
|
||||
use image::{DynamicImage, EncodableLayout, ImageBuffer, PixelWithColorType};
|
||||
use poise::serenity_prelude::Timestamp;
|
||||
|
||||
use crate::context::Error;
|
||||
|
||||
#[inline]
|
||||
fn should_save_debug_images() -> bool {
|
||||
env::var("SHIMMERING_DEBUG_IMGS")
|
||||
.map(|s| s == "1")
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn debug_image_log(image: &DynamicImage) -> Result<(), Error> {
|
||||
if should_save_debug_images() {
|
||||
image.save(format!("./logs/{}.png", Timestamp::now()))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn debug_image_buffer_log<P, C>(image: &ImageBuffer<P, C>) -> Result<(), Error>
|
||||
where
|
||||
P: PixelWithColorType,
|
||||
[P::Subpixel]: EncodableLayout,
|
||||
C: Deref<Target = [P::Subpixel]>,
|
||||
{
|
||||
if should_save_debug_images() {
|
||||
image.save(format!("./logs/{}.png", Timestamp::now()))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -11,6 +11,7 @@ mod bitmap;
|
|||
mod commands;
|
||||
mod context;
|
||||
mod levenshtein;
|
||||
mod logs;
|
||||
mod recognition;
|
||||
mod transform;
|
||||
mod user;
|
||||
|
|
|
|||
289
src/recognition/hyperglass.rs
Normal file
289
src/recognition/hyperglass.rs
Normal file
|
|
@ -0,0 +1,289 @@
|
|||
use freetype::Face;
|
||||
use image::{DynamicImage, ImageBuffer, Luma};
|
||||
use imageproc::{
|
||||
contrast::{threshold, ThresholdType},
|
||||
region_labelling::{connected_components, Connectivity},
|
||||
};
|
||||
use num::traits::Euclid;
|
||||
|
||||
use crate::{
|
||||
bitmap::{Align, BitmapCanvas, Color, TextStyle},
|
||||
context::Error,
|
||||
logs::{debug_image_buffer_log, debug_image_log},
|
||||
};
|
||||
|
||||
///! Hyperglass my own specialized OCR system
|
||||
|
||||
// {{{ ConponentVec
|
||||
/// How many sub-segments to split each side into
|
||||
const SPLIT_FACTOR: u32 = 5;
|
||||
const IMAGE_VEC_DIM: usize = (SPLIT_FACTOR * SPLIT_FACTOR) as usize;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct ComponentVec {
|
||||
chunks: [f32; IMAGE_VEC_DIM],
|
||||
}
|
||||
|
||||
impl ComponentVec {
|
||||
// {{{ (Component => vector) encoding
|
||||
fn from_component(components: &ComponentsWithBounds, component: u32) -> Result<Self, Error> {
|
||||
let mut chunks = [0.0; IMAGE_VEC_DIM];
|
||||
let bounds = components
|
||||
.bounds
|
||||
.get(component as usize - 1)
|
||||
.and_then(|o| o.as_ref())
|
||||
.ok_or_else(|| "Missing bounds for given connected component")?;
|
||||
|
||||
for i in 0..(SPLIT_FACTOR * SPLIT_FACTOR) {
|
||||
let (iy, ix) = i.div_rem_euclid(&SPLIT_FACTOR);
|
||||
|
||||
let x_start = bounds.x_min + ix * components.max_width / SPLIT_FACTOR;
|
||||
let x_end = bounds.x_min + (ix + 1) * components.max_width / SPLIT_FACTOR;
|
||||
let y_start = bounds.y_min + iy * components.max_height / SPLIT_FACTOR;
|
||||
let y_end = bounds.y_min + (iy + 1) * components.max_height / SPLIT_FACTOR;
|
||||
let mut count = 0;
|
||||
|
||||
for x in x_start..x_end {
|
||||
for y in y_start..y_end {
|
||||
if let Some(p) = components.components.get_pixel_checked(x, y)
|
||||
&& p.0[0] == component
|
||||
{
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let size = (x_end + 1 - x_start) * (y_end + 1 - y_start);
|
||||
|
||||
if size == 0 {
|
||||
return Err(format!(
|
||||
"Got zero size for chunk [{x_start},{x_end}]x[{y_start},{y_end}]"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
chunks[i as usize] = count as f32 / size as f32;
|
||||
|
||||
// print!("{} ", chunks[i as usize]);
|
||||
// if i % SPLIT_FACTOR == SPLIT_FACTOR - 1 {
|
||||
// print!("\n");
|
||||
// }
|
||||
}
|
||||
|
||||
let mut result = Self { chunks };
|
||||
result.normalise();
|
||||
Ok(result)
|
||||
}
|
||||
// }}}
|
||||
// {{{ Distance
|
||||
#[inline]
|
||||
fn distance_squared_to(&self, other: &Self) -> f32 {
|
||||
let mut total = 0.0;
|
||||
|
||||
for i in 0..IMAGE_VEC_DIM {
|
||||
let d = self.chunks[i] - other.chunks[i];
|
||||
total += d * d;
|
||||
}
|
||||
|
||||
total
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn norm_squared(&self) -> f32 {
|
||||
let mut total = 0.0;
|
||||
|
||||
for i in 0..IMAGE_VEC_DIM {
|
||||
total += self.chunks[i] * self.chunks[i];
|
||||
}
|
||||
|
||||
total
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn normalise(&mut self) {
|
||||
let len = self.norm_squared().sqrt();
|
||||
|
||||
for i in 0..IMAGE_VEC_DIM {
|
||||
self.chunks[i] /= len;
|
||||
}
|
||||
}
|
||||
// }}}
|
||||
}
|
||||
// }}}
|
||||
// {{{ Component bounds
|
||||
#[derive(Clone, Copy)]
|
||||
struct ComponentBounds {
|
||||
x_min: u32,
|
||||
y_min: u32,
|
||||
x_max: u32,
|
||||
y_max: u32,
|
||||
}
|
||||
|
||||
struct ComponentsWithBounds {
|
||||
components: ImageBuffer<Luma<u32>, Vec<u32>>,
|
||||
|
||||
// NOTE: the index is (the id of the component) - 1
|
||||
// This is because the zero component represents the background,
|
||||
// but we don't want to waste a place in this vector.
|
||||
bounds: Vec<Option<ComponentBounds>>,
|
||||
|
||||
max_width: u32,
|
||||
max_height: u32,
|
||||
|
||||
/// Stores the indices of `self.bounds` sorted based on their min position.
|
||||
bounds_by_position: Vec<usize>,
|
||||
}
|
||||
|
||||
impl ComponentsWithBounds {
|
||||
fn from_image(image: &DynamicImage) -> Result<Self, Error> {
|
||||
let image = threshold(&image.to_luma8(), 100, ThresholdType::Binary);
|
||||
debug_image_buffer_log(&image)?;
|
||||
|
||||
let background = Luma([u8::MAX]);
|
||||
let components = connected_components(&image, Connectivity::Eight, background);
|
||||
|
||||
let mut bounds: Vec<Option<ComponentBounds>> = Vec::new();
|
||||
for x in 0..components.width() {
|
||||
for y in 0..components.height() {
|
||||
// {{{ Retrieve pixel if it's not backround
|
||||
let component = components[(x, y)].0[0];
|
||||
if component == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let index = component as usize - 1;
|
||||
if index >= bounds.len() {
|
||||
bounds.resize(index + 1, None);
|
||||
}
|
||||
// }}}
|
||||
// {{{ Update bounds
|
||||
if let Some(bounds) = (&mut bounds)[index].as_mut() {
|
||||
bounds.x_min = bounds.x_min.min(x);
|
||||
bounds.x_max = bounds.x_max.max(x);
|
||||
bounds.y_min = bounds.y_min.min(y);
|
||||
bounds.y_max = bounds.y_max.max(y);
|
||||
} else {
|
||||
bounds[index] = Some(ComponentBounds {
|
||||
x_min: x,
|
||||
x_max: x,
|
||||
y_min: y,
|
||||
y_max: y,
|
||||
});
|
||||
}
|
||||
// }}}
|
||||
}
|
||||
}
|
||||
|
||||
// {{{ Remove components that are too large
|
||||
for bound in &mut bounds {
|
||||
if bound.map_or(false, |b| (b.x_max - b.x_min) >= 9 * image.width() / 10) {
|
||||
*bound = None;
|
||||
}
|
||||
}
|
||||
// }}}
|
||||
// {{{ Compute max width/height
|
||||
let max_width = bounds
|
||||
.iter()
|
||||
.filter_map(|o| o.as_ref())
|
||||
.map(|b| b.x_max - b.x_min)
|
||||
.max()
|
||||
.ok_or_else(|| "No connected components found")?;
|
||||
let max_height = bounds
|
||||
.iter()
|
||||
.filter_map(|o| o.as_ref())
|
||||
.map(|b| b.y_max - b.y_min)
|
||||
.max()
|
||||
.ok_or_else(|| "No connected components found")?;
|
||||
// }}}
|
||||
|
||||
let mut bounds_by_position: Vec<usize> = (0..(bounds.len()))
|
||||
.filter(|i| bounds[*i].is_some())
|
||||
.collect();
|
||||
bounds_by_position.sort_by_key(|i| bounds[*i].unwrap().x_min);
|
||||
|
||||
Ok(Self {
|
||||
components,
|
||||
bounds,
|
||||
max_width,
|
||||
max_height,
|
||||
bounds_by_position,
|
||||
})
|
||||
}
|
||||
}
|
||||
// }}}
|
||||
// {{{ Char measurements
|
||||
pub struct CharMeasurements {
|
||||
chars: Vec<(char, ComponentVec)>,
|
||||
}
|
||||
|
||||
impl CharMeasurements {
|
||||
// {{{ Creation
|
||||
pub fn from_text(face: &mut Face, string: &str, weight: Option<u32>) -> Result<Self, Error> {
|
||||
// These are bad estimates lol
|
||||
let char_w = 35;
|
||||
let char_h = 60;
|
||||
|
||||
let mut canvas = BitmapCanvas::new(10 + char_w * string.len() as u32, char_h + 10);
|
||||
canvas.text(
|
||||
(5, 5),
|
||||
face,
|
||||
TextStyle {
|
||||
stroke: None,
|
||||
drop_shadow: None,
|
||||
align: (Align::Start, Align::Start),
|
||||
size: char_h,
|
||||
color: Color::BLACK,
|
||||
weight: None,
|
||||
},
|
||||
&string,
|
||||
)?;
|
||||
let buffer = ImageBuffer::from_raw(canvas.width, canvas.height(), canvas.buffer.to_vec())
|
||||
.ok_or_else(|| "Failed to turn buffer into canvas")?;
|
||||
let image = DynamicImage::ImageRgb8(buffer);
|
||||
|
||||
debug_image_log(&image)?;
|
||||
|
||||
let components = ComponentsWithBounds::from_image(&image)?;
|
||||
|
||||
let mut chars = Vec::with_capacity(string.len());
|
||||
for (i, char) in string.chars().enumerate() {
|
||||
chars.push((
|
||||
char,
|
||||
ComponentVec::from_component(
|
||||
&components,
|
||||
components.bounds_by_position[i] as u32 + 1,
|
||||
)?,
|
||||
))
|
||||
}
|
||||
|
||||
Ok(Self { chars })
|
||||
}
|
||||
// }}}
|
||||
// {{{ Recognition
|
||||
pub fn recognise(&self, image: &DynamicImage) -> Result<String, Error> {
|
||||
let components = ComponentsWithBounds::from_image(image)?;
|
||||
let mut result = String::new();
|
||||
for i in &components.bounds_by_position {
|
||||
let vec = ComponentVec::from_component(&components, *i as u32 + 1)?;
|
||||
|
||||
let best_match = self
|
||||
.chars
|
||||
.iter()
|
||||
.map(|(i, v)| (*i, v, v.distance_squared_to(&vec)))
|
||||
.min_by(|(_, _, d1), (_, _, d2)| {
|
||||
d1.partial_cmp(d2).expect("NaN distance encountered")
|
||||
})
|
||||
.map(|(i, _, d)| (d.sqrt(), i))
|
||||
.ok_or_else(|| "No chars in cache")?;
|
||||
|
||||
// println!("char '{}', distance {}", best_match.1, best_match.0);
|
||||
if best_match.0 <= (IMAGE_VEC_DIM * 10) as f32 {
|
||||
result.push(best_match.1);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
// }}}
|
||||
}
|
||||
// }}}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
pub mod fuzzy_song_name;
|
||||
pub mod hyperglass;
|
||||
pub mod recognize;
|
||||
pub mod ui;
|
||||
|
|
|
|||
|
|
@ -1,14 +1,12 @@
|
|||
use std::fmt::Display;
|
||||
use std::io::Cursor;
|
||||
use std::str::FromStr;
|
||||
use std::{env, fs};
|
||||
|
||||
use hypertesseract::{PageSegMode, Tesseract};
|
||||
use image::imageops::{resize, FilterType};
|
||||
use image::{DynamicImage, GenericImageView, RgbaImage};
|
||||
use image::imageops::FilterType;
|
||||
use image::{DynamicImage, GenericImageView};
|
||||
use image::{ImageBuffer, Rgba};
|
||||
use num::integer::Roots;
|
||||
use poise::serenity_prelude::{CreateAttachment, CreateEmbed, CreateMessage, Timestamp};
|
||||
use poise::serenity_prelude::{CreateAttachment, CreateEmbed, CreateMessage};
|
||||
|
||||
use crate::arcaea::chart::{Chart, Difficulty, Song, DIFFICULTY_MENU_PIXEL_COLORS};
|
||||
use crate::arcaea::jacket::IMAGE_VEC_DIM;
|
||||
|
|
@ -16,6 +14,7 @@ use crate::arcaea::score::Score;
|
|||
use crate::bitmap::{Color, Rect};
|
||||
use crate::context::{Context, Error, UserContext};
|
||||
use crate::levenshtein::edit_distance;
|
||||
use crate::logs::debug_image_buffer_log;
|
||||
use crate::recognition::fuzzy_song_name::guess_chart_name;
|
||||
use crate::recognition::ui::{
|
||||
ScoreScreenRect, SongSelectRect, UIMeasurementRect, UIMeasurementRect::*,
|
||||
|
|
@ -47,25 +46,6 @@ impl ImageAnalyzer {
|
|||
}
|
||||
|
||||
// {{{ Crop
|
||||
#[inline]
|
||||
fn should_save_debug_images() -> bool {
|
||||
env::var("SHIMMERING_DEBUG_IMGS")
|
||||
.map(|s| s == "1")
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn save_image(&mut self, image: &RgbaImage) -> Result<(), Error> {
|
||||
self.clear();
|
||||
let mut cursor = Cursor::new(&mut self.bytes);
|
||||
image.write_to(&mut cursor, image::ImageFormat::Png)?;
|
||||
|
||||
if Self::should_save_debug_images() {
|
||||
fs::write(format!("./logs/{}.png", Timestamp::now()), &self.bytes)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn crop(&mut self, image: &DynamicImage, rect: Rect) -> ImageBuffer<Rgba<u8>, Vec<u8>> {
|
||||
image
|
||||
|
|
@ -84,9 +64,7 @@ impl ImageAnalyzer {
|
|||
self.last_rect = Some((ui_rect, rect));
|
||||
|
||||
let result = self.crop(image, rect);
|
||||
if Self::should_save_debug_images() {
|
||||
self.save_image(&result).unwrap();
|
||||
}
|
||||
debug_image_buffer_log(&result)?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
|
@ -97,18 +75,17 @@ impl ImageAnalyzer {
|
|||
ctx: &UserContext,
|
||||
image: &DynamicImage,
|
||||
ui_rect: UIMeasurementRect,
|
||||
size: impl FnOnce(Rect) -> (u32, u32),
|
||||
size: (u32, u32),
|
||||
) -> Result<ImageBuffer<Rgba<u8>, Vec<u8>>, Error> {
|
||||
let rect = ctx.ui_measurements.interpolate(ui_rect, image)?;
|
||||
let size = size(rect);
|
||||
self.last_rect = Some((ui_rect, rect));
|
||||
|
||||
let result = self.crop(image, rect);
|
||||
let result = resize(&result, size.0, size.1, FilterType::Nearest);
|
||||
let result = DynamicImage::ImageRgba8(result)
|
||||
.resize(size.0, size.1, FilterType::Nearest)
|
||||
.into_rgba8();
|
||||
|
||||
if Self::should_save_debug_images() {
|
||||
self.save_image(&result).unwrap();
|
||||
}
|
||||
debug_image_buffer_log(&result)?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
|
@ -130,8 +107,7 @@ impl ImageAnalyzer {
|
|||
));
|
||||
|
||||
if let Some((ui_rect, rect)) = self.last_rect {
|
||||
let cropped = self.crop(image, rect);
|
||||
self.save_image(&cropped)?;
|
||||
self.crop(image, rect);
|
||||
|
||||
let bytes = std::mem::take(&mut self.bytes);
|
||||
let error_attachement = CreateAttachment::bytes(bytes, filename);
|
||||
|
|
@ -161,14 +137,7 @@ impl ImageAnalyzer {
|
|||
note_count: Option<u32>,
|
||||
image: &DynamicImage,
|
||||
kind: ScoreKind,
|
||||
) -> Result<Vec<Score>, Error> {
|
||||
// yes, this was painfully hand-picked
|
||||
let desired_height = 100;
|
||||
let x_scaling_factor = match kind {
|
||||
ScoreKind::SongSelect => 1.0,
|
||||
ScoreKind::ScoreScreen => 0.666,
|
||||
};
|
||||
|
||||
) -> Result<Score, Error> {
|
||||
let image = self.interp_crop_resize(
|
||||
ctx,
|
||||
image,
|
||||
|
|
@ -176,123 +145,37 @@ impl ImageAnalyzer {
|
|||
ScoreKind::SongSelect => SongSelect(SongSelectRect::Score),
|
||||
ScoreKind::ScoreScreen => ScoreScreen(ScoreScreenRect::Score),
|
||||
},
|
||||
|rect| {
|
||||
(
|
||||
(rect.width as f32 * desired_height as f32 / rect.height as f32
|
||||
* x_scaling_factor) as u32,
|
||||
desired_height,
|
||||
)
|
||||
},
|
||||
(u32::MAX, 100),
|
||||
)?;
|
||||
|
||||
let mut results = vec![];
|
||||
for mode in [
|
||||
PageSegMode::SingleWord,
|
||||
PageSegMode::RawLine,
|
||||
PageSegMode::SingleLine,
|
||||
PageSegMode::SparseText,
|
||||
PageSegMode::SingleBlock,
|
||||
] {
|
||||
let result: Result<_, Error> = try {
|
||||
// {{{ Read score using tesseract
|
||||
let text = Tesseract::builder()
|
||||
.language(hypertesseract::Language::English)
|
||||
.whitelist_str("0123456789'/")?
|
||||
.page_seg_mode(mode)
|
||||
.assume_numeric_input()
|
||||
.build()?
|
||||
.load_image(&image)?
|
||||
.recognize()?
|
||||
.get_text()?;
|
||||
let measurements = match kind {
|
||||
ScoreKind::SongSelect => &ctx.exo_measurements,
|
||||
ScoreKind::ScoreScreen => &ctx.geosans_measurements,
|
||||
};
|
||||
|
||||
let text: String = text
|
||||
.trim()
|
||||
.chars()
|
||||
.map(|char| if char == '/' { '7' } else { char })
|
||||
.filter(|char| *char != ' ' && *char != '\'')
|
||||
.collect();
|
||||
let result = Score(
|
||||
measurements
|
||||
.recognise(&DynamicImage::ImageRgba8(image))?
|
||||
.chars()
|
||||
.filter(|c| *c != '\'')
|
||||
.collect::<String>()
|
||||
.parse()?,
|
||||
);
|
||||
|
||||
let score = u32::from_str_radix(&text, 10)?;
|
||||
Score(score)
|
||||
// }}}
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(result) => {
|
||||
results.push(result.0);
|
||||
}
|
||||
Err(err) => {
|
||||
println!("OCR score result error: {}", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// {{{ Score correction
|
||||
// The OCR sometimes fails to read "74" with the arcaea font,
|
||||
// so we try to detect that and fix it
|
||||
loop {
|
||||
let old_stack_len = results.len();
|
||||
println!("Results {:?}", results);
|
||||
results = results
|
||||
.iter()
|
||||
.flat_map(|result| {
|
||||
// If the length is correct, we are good to go!
|
||||
if *result >= 8_000_000 {
|
||||
vec![*result]
|
||||
} else {
|
||||
let mut results = vec![];
|
||||
for i in [0, 1, 3, 4] {
|
||||
let d = 10u32.pow(i);
|
||||
if (*result / d) % 10 == 4 && (*result / d) % 100 != 74 {
|
||||
let n = d * 10;
|
||||
results.push((*result / n) * n * 10 + 7 * n + (*result % n));
|
||||
}
|
||||
}
|
||||
|
||||
results
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
if old_stack_len == results.len() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// }}}
|
||||
// {{{ Return score if consensus exists
|
||||
// 1. Discard scores that are known to be impossible
|
||||
let mut results: Vec<_> = results
|
||||
.into_iter()
|
||||
.filter(|result| {
|
||||
8_000_000 <= *result
|
||||
&& *result <= 10_010_000
|
||||
&& note_count
|
||||
.map(|note_count| {
|
||||
let (zeta, shinies, score_units) = Score(*result).analyse(note_count);
|
||||
8_000_000 <= zeta.0
|
||||
&& zeta.0 <= 10_000_000 && shinies <= note_count
|
||||
&& score_units <= 2 * note_count
|
||||
})
|
||||
.unwrap_or(true)
|
||||
})
|
||||
.map(|r| Score(r))
|
||||
.collect();
|
||||
println!("Results {:?}", results);
|
||||
|
||||
// 2. Look for consensus
|
||||
for result in results.iter() {
|
||||
if results.iter().filter(|e| **e == *result).count() > results.len() / 2 {
|
||||
return Ok(vec![*result]);
|
||||
}
|
||||
if result.0 <= 10_010_000
|
||||
&& note_count.map_or(true, |note_count| {
|
||||
let (zeta, shinies, score_units) = result.analyse(note_count);
|
||||
8_000_000 <= zeta.0
|
||||
&& zeta.0 <= 10_000_000
|
||||
&& shinies <= note_count
|
||||
&& score_units <= 2 * note_count
|
||||
}) {
|
||||
Ok(result)
|
||||
} else {
|
||||
Err(format!("Score {result} is not vaild").into())
|
||||
}
|
||||
// }}}
|
||||
|
||||
// If there's no consensus, we return everything
|
||||
results.sort();
|
||||
results.dedup();
|
||||
println!("Results {:?}", results);
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
// }}}
|
||||
// {{{ Read difficulty
|
||||
|
|
@ -335,24 +218,25 @@ impl ImageAnalyzer {
|
|||
return Ok(min.1);
|
||||
}
|
||||
|
||||
let mut ocr = Tesseract::builder()
|
||||
let (text, conf) = Tesseract::builder()
|
||||
.language(hypertesseract::Language::English)
|
||||
.page_seg_mode(PageSegMode::RawLine)
|
||||
.build()?;
|
||||
.build()?
|
||||
.recognize_text_cloned_with_conf(&self.interp_crop(
|
||||
ctx,
|
||||
image,
|
||||
ScoreScreen(ScoreScreenRect::Difficulty),
|
||||
)?)?;
|
||||
|
||||
ocr.load_image(&self.interp_crop(ctx, image, ScoreScreen(ScoreScreenRect::Difficulty))?)?
|
||||
.recognize()?;
|
||||
|
||||
let text: &str = &ocr.get_text()?;
|
||||
let text = text.trim().to_lowercase();
|
||||
|
||||
// let conf = t.mean_text_conf();
|
||||
// if conf < 10 && conf != 0 {
|
||||
// Err(format!(
|
||||
// "Difficulty text is not readable (confidence = {}, text = {}).",
|
||||
// conf, text
|
||||
// ))?;
|
||||
// }
|
||||
if conf < 10 && conf != 0 {
|
||||
return Err(format!(
|
||||
"Difficulty text is not readable (confidence = {}, text = {}).",
|
||||
conf, text
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
let difficulty = Difficulty::DIFFICULTIES
|
||||
.iter()
|
||||
|
|
@ -370,23 +254,21 @@ impl ImageAnalyzer {
|
|||
ctx: &UserContext,
|
||||
image: &DynamicImage,
|
||||
) -> Result<ScoreKind, Error> {
|
||||
let text = Tesseract::builder()
|
||||
let (text, conf) = Tesseract::builder()
|
||||
.language(hypertesseract::Language::English)
|
||||
.page_seg_mode(PageSegMode::RawLine)
|
||||
.build()?
|
||||
.load_image(&self.interp_crop(ctx, image, PlayKind)?)?
|
||||
.recognize()?
|
||||
.get_text()?
|
||||
.trim()
|
||||
.to_string();
|
||||
.recognize_text_cloned_with_conf(&self.interp_crop(ctx, image, PlayKind)?)?;
|
||||
|
||||
// let conf = t.mean_text_conf();
|
||||
// if conf < 10 && conf != 0 {
|
||||
// Err(format!(
|
||||
// "Score kind text is not readable (confidence = {}, text = {}).",
|
||||
// conf, text
|
||||
// ))?;
|
||||
// }
|
||||
let text = text.trim().to_string();
|
||||
|
||||
if conf < 10 && conf != 0 {
|
||||
return Err(format!(
|
||||
"Score kind text is not readable (confidence = {}, text = {}).",
|
||||
conf, text
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
let result = if edit_distance(&text, "Result") < edit_distance(&text, "Select a song") {
|
||||
ScoreKind::ScoreScreen
|
||||
|
|
@ -404,23 +286,25 @@ impl ImageAnalyzer {
|
|||
image: &DynamicImage,
|
||||
difficulty: Difficulty,
|
||||
) -> Result<(&'a Song, &'a Chart), Error> {
|
||||
let text = Tesseract::builder()
|
||||
let (text, conf) = Tesseract::builder()
|
||||
.language(hypertesseract::Language::English)
|
||||
.page_seg_mode(PageSegMode::SingleLine)
|
||||
.whitelist_str("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789,.()- ")?
|
||||
.build()?
|
||||
.load_image(&self.interp_crop(ctx, image, ScoreScreen(ScoreScreenRect::Title))?)?
|
||||
.recognize()?
|
||||
.get_text()?;
|
||||
.recognize_text_cloned_with_conf(&self.interp_crop(
|
||||
ctx,
|
||||
image,
|
||||
ScoreScreen(ScoreScreenRect::Title),
|
||||
)?)?;
|
||||
|
||||
// let conf = t.mean_text_conf();
|
||||
// if conf < 20 && conf != 0 {
|
||||
// Err(format!(
|
||||
// "Title text is not readable (confidence = {}, text = {}).",
|
||||
// conf,
|
||||
// raw_text.trim()
|
||||
// ))?;
|
||||
// }
|
||||
if conf < 20 && conf != 0 {
|
||||
return Err(format!(
|
||||
"Title text is not readable (confidence = {}, text = {}).",
|
||||
conf,
|
||||
text.trim()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
guess_chart_name(&text, &ctx.song_cache, Some(difficulty), false)
|
||||
}
|
||||
|
|
@ -478,23 +362,19 @@ impl ImageAnalyzer {
|
|||
ctx: &UserContext,
|
||||
image: &DynamicImage,
|
||||
) -> Result<(u32, u32, u32), Error> {
|
||||
let mut ocr = Tesseract::builder()
|
||||
.language(hypertesseract::Language::English)
|
||||
.page_seg_mode(PageSegMode::SparseText)
|
||||
.whitelist_str("0123456789")?
|
||||
.assume_numeric_input()
|
||||
.build()?;
|
||||
|
||||
let mut out = [0; 3];
|
||||
|
||||
use ScoreScreenRect::*;
|
||||
static KINDS: [ScoreScreenRect; 3] = [Pure, Far, Lost];
|
||||
|
||||
for i in 0..3 {
|
||||
let text = ocr
|
||||
.load_image(&self.interp_crop(ctx, image, ScoreScreen(KINDS[i]))?)?
|
||||
.recognize()?
|
||||
.get_text()?;
|
||||
let text = Tesseract::builder()
|
||||
.language(hypertesseract::Language::English)
|
||||
.page_seg_mode(PageSegMode::SparseText)
|
||||
.whitelist_str("0123456789")?
|
||||
.assume_numeric_input()
|
||||
.build()?
|
||||
.recognize_text_cloned(&self.interp_crop(ctx, image, ScoreScreen(KINDS[i]))?)?;
|
||||
|
||||
println!("Raw '{}'", text.trim());
|
||||
out[i] = u32::from_str(&text.trim()).unwrap_or(0);
|
||||
|
|
@ -510,26 +390,28 @@ impl ImageAnalyzer {
|
|||
ctx: &'a UserContext,
|
||||
image: &DynamicImage,
|
||||
) -> Result<u32, Error> {
|
||||
let text = Tesseract::builder()
|
||||
let (text, conf) = Tesseract::builder()
|
||||
.language(hypertesseract::Language::English)
|
||||
.page_seg_mode(PageSegMode::SingleLine)
|
||||
.whitelist_str("0123456789")?
|
||||
.assume_numeric_input()
|
||||
.build()?
|
||||
.load_image(&self.interp_crop(ctx, image, ScoreScreen(ScoreScreenRect::MaxRecall))?)?
|
||||
.recognize()?
|
||||
.get_text()?;
|
||||
.recognize_text_cloned_with_conf(&self.interp_crop(
|
||||
ctx,
|
||||
image,
|
||||
ScoreScreen(ScoreScreenRect::MaxRecall),
|
||||
)?)?;
|
||||
|
||||
let max_recall = u32::from_str_radix(text.trim(), 10)?;
|
||||
|
||||
// let conf = t.mean_text_conf();
|
||||
// if conf < 20 && conf != 0 {
|
||||
// Err(format!(
|
||||
// "Title text is not readable (confidence = {}, text = {}).",
|
||||
// conf,
|
||||
// raw_text.trim()
|
||||
// ))?;
|
||||
// }
|
||||
if conf < 20 && conf != 0 {
|
||||
return Err(format!(
|
||||
"Title text is not readable (confidence = {}, text = {}).",
|
||||
conf,
|
||||
text.trim()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
Ok(max_recall)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue