Skip to content

Commit 8c54d5b

Browse files
Florobalexcrichton
authored andcommitted
core: Move Hangul decomposition into unicode.rs
1 parent 2f71b72 commit 8c54d5b

File tree

3 files changed

+110
-69
lines changed

3 files changed

+110
-69
lines changed

src/etc/unicode.py

Lines changed: 58 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -321,17 +321,24 @@ def emit_core_decomp_module(f, canon, compat):
321321
format_table_content(f, data, 8)
322322
f.write("\n ];\n\n")
323323

324-
f.write(" pub fn canonical(c: char, i: |char|) "
325-
+ "{ d(c, i, false); }\n\n")
326-
f.write(" pub fn compatibility(c: char, i: |char|) "
327-
+"{ d(c, i, true); }\n\n")
328-
f.write(" fn d(c: char, i: |char|, k: bool) {\n")
329-
f.write(" use iter::Iterator;\n");
324+
f.write("""
325+
pub fn decompose_canonical(c: char, i: |char|) { d(c, i, false); }
330326
331-
f.write(" if c <= '\\x7f' { i(c); return; }\n")
327+
pub fn decompose_compatible(c: char, i: |char|) { d(c, i, true); }
332328
333-
# First check the canonical decompositions
334-
f.write("""
329+
fn d(c: char, i: |char|, k: bool) {
330+
use iter::Iterator;
331+
332+
// 7-bit ASCII never decomposes
333+
if c <= '\\x7f' { i(c); return; }
334+
335+
// Perform decomposition for Hangul
336+
if (c as u32) >= S_BASE && (c as u32) < (S_BASE + S_COUNT) {
337+
decompose_hangul(c, i);
338+
return;
339+
}
340+
341+
// First check the canonical decompositions
335342
match bsearch_table(c, canonical_table) {
336343
Some(canon) => {
337344
for x in canon.iter() {
@@ -340,13 +347,12 @@ def emit_core_decomp_module(f, canon, compat):
340347
return;
341348
}
342349
None => ()
343-
}\n\n""")
350+
}
344351
345-
# Bottom out if we're not doing compat.
346-
f.write(" if !k { i(c); return; }\n")
352+
// Bottom out if we're not doing compat.
353+
if !k { i(c); return; }
347354
348-
# Then check the compatibility decompositions
349-
f.write("""
355+
// Then check the compatibility decompositions
350356
match bsearch_table(c, compatibility_table) {
351357
Some(compat) => {
352358
for x in compat.iter() {
@@ -355,12 +361,45 @@ def emit_core_decomp_module(f, canon, compat):
355361
return;
356362
}
357363
None => ()
358-
}\n\n""")
364+
}
359365
360-
# Finally bottom out.
361-
f.write(" i(c);\n")
362-
f.write(" }\n")
363-
f.write("}\n\n")
366+
// Finally bottom out.
367+
i(c);
368+
}
369+
370+
// Constants from Unicode 6.2.0 Section 3.12 Conjoining Jamo Behavior
371+
static S_BASE: u32 = 0xAC00;
372+
static L_BASE: u32 = 0x1100;
373+
static V_BASE: u32 = 0x1161;
374+
static T_BASE: u32 = 0x11A7;
375+
static L_COUNT: u32 = 19;
376+
static V_COUNT: u32 = 21;
377+
static T_COUNT: u32 = 28;
378+
static N_COUNT: u32 = (V_COUNT * T_COUNT);
379+
static S_COUNT: u32 = (L_COUNT * N_COUNT);
380+
381+
// Decompose a precomposed Hangul syllable
382+
fn decompose_hangul(s: char, f: |char|) {
383+
use cast::transmute;
384+
385+
let si = s as u32 - S_BASE;
386+
387+
let li = si / N_COUNT;
388+
unsafe {
389+
f(transmute(L_BASE + li));
390+
391+
let vi = (si % N_COUNT) / T_COUNT;
392+
f(transmute(V_BASE + vi));
393+
394+
let ti = si % T_COUNT;
395+
if ti > 0 {
396+
f(transmute(T_BASE + ti));
397+
}
398+
}
399+
}
400+
}
401+
402+
""")
364403

365404
def emit_std_decomp_module(f, combine):
366405
f.write("pub mod decompose {\n");

src/libcore/char.rs

Lines changed: 6 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,12 @@
2727
use mem::transmute;
2828
use option::{None, Option, Some};
2929
use iter::{Iterator, range_step};
30-
use unicode::{derived_property, property, general_category, decompose, conversions};
30+
use unicode::{derived_property, property, general_category, conversions};
31+
32+
/// Returns the canonical decomposition of a character.
33+
pub use unicode::decompose::decompose_canonical;
34+
/// Returns the compatibility decomposition of a character.
35+
pub use unicode::decompose::decompose_compatible;
3136

3237
#[cfg(not(test))] use cmp::{Eq, Ord, TotalEq, TotalOrd, Ordering};
3338
#[cfg(not(test))] use default::Default;
@@ -285,53 +290,6 @@ pub fn from_digit(num: uint, radix: uint) -> Option<char> {
285290
}
286291
}
287292

288-
// Constants from Unicode 6.2.0 Section 3.12 Conjoining Jamo Behavior
289-
static S_BASE: u32 = 0xAC00;
290-
static L_BASE: u32 = 0x1100;
291-
static V_BASE: u32 = 0x1161;
292-
static T_BASE: u32 = 0x11A7;
293-
static L_COUNT: u32 = 19;
294-
static V_COUNT: u32 = 21;
295-
static T_COUNT: u32 = 28;
296-
static N_COUNT: u32 = (V_COUNT * T_COUNT);
297-
static S_COUNT: u32 = (L_COUNT * N_COUNT);
298-
299-
// Decompose a precomposed Hangul syllable
300-
fn decompose_hangul(s: char, f: |char|) {
301-
let si = s as u32 - S_BASE;
302-
303-
let li = si / N_COUNT;
304-
unsafe {
305-
f(transmute(L_BASE + li));
306-
307-
let vi = (si % N_COUNT) / T_COUNT;
308-
f(transmute(V_BASE + vi));
309-
310-
let ti = si % T_COUNT;
311-
if ti > 0 {
312-
f(transmute(T_BASE + ti));
313-
}
314-
}
315-
}
316-
317-
/// Returns the canonical decomposition of a character
318-
pub fn decompose_canonical(c: char, f: |char|) {
319-
if (c as u32) < S_BASE || (c as u32) >= (S_BASE + S_COUNT) {
320-
decompose::canonical(c, f);
321-
} else {
322-
decompose_hangul(c, f);
323-
}
324-
}
325-
326-
/// Returns the compatibility decomposition of a character
327-
pub fn decompose_compatible(c: char, f: |char|) {
328-
if (c as u32) < S_BASE || (c as u32) >= (S_BASE + S_COUNT) {
329-
decompose::compatibility(c, f);
330-
} else {
331-
decompose_hangul(c, f);
332-
}
333-
}
334-
335293
///
336294
/// Returns the hexadecimal Unicode escape of a `char`
337295
///

src/libcore/unicode.rs

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2121,14 +2121,24 @@ pub mod decompose {
21212121
&['\u53ef'])
21222122
];
21232123

2124-
pub fn canonical(c: char, i: |char|) { d(c, i, false); }
21252124

2126-
pub fn compatibility(c: char, i: |char|) { d(c, i, true); }
2125+
pub fn decompose_canonical(c: char, i: |char|) { d(c, i, false); }
2126+
2127+
pub fn decompose_compatible(c: char, i: |char|) { d(c, i, true); }
21272128

21282129
fn d(c: char, i: |char|, k: bool) {
21292130
use iter::Iterator;
2131+
2132+
// 7-bit ASCII never decomposes
21302133
if c <= '\x7f' { i(c); return; }
21312134

2135+
// Perform decomposition for Hangul
2136+
if (c as u32) >= S_BASE && (c as u32) < (S_BASE + S_COUNT) {
2137+
decompose_hangul(c, i);
2138+
return;
2139+
}
2140+
2141+
// First check the canonical decompositions
21322142
match bsearch_table(c, canonical_table) {
21332143
Some(canon) => {
21342144
for x in canon.iter() {
@@ -2139,8 +2149,10 @@ pub mod decompose {
21392149
None => ()
21402150
}
21412151

2152+
// Bottom out if we're not doing compat.
21422153
if !k { i(c); return; }
21432154

2155+
// Then check the compatibility decompositions
21442156
match bsearch_table(c, compatibility_table) {
21452157
Some(compat) => {
21462158
for x in compat.iter() {
@@ -2151,8 +2163,40 @@ pub mod decompose {
21512163
None => ()
21522164
}
21532165

2166+
// Finally bottom out.
21542167
i(c);
21552168
}
2169+
2170+
// Constants from Unicode 6.2.0 Section 3.12 Conjoining Jamo Behavior
2171+
static S_BASE: u32 = 0xAC00;
2172+
static L_BASE: u32 = 0x1100;
2173+
static V_BASE: u32 = 0x1161;
2174+
static T_BASE: u32 = 0x11A7;
2175+
static L_COUNT: u32 = 19;
2176+
static V_COUNT: u32 = 21;
2177+
static T_COUNT: u32 = 28;
2178+
static N_COUNT: u32 = (V_COUNT * T_COUNT);
2179+
static S_COUNT: u32 = (L_COUNT * N_COUNT);
2180+
2181+
// Decompose a precomposed Hangul syllable
2182+
fn decompose_hangul(s: char, f: |char|) {
2183+
use mem::transmute;
2184+
2185+
let si = s as u32 - S_BASE;
2186+
2187+
let li = si / N_COUNT;
2188+
unsafe {
2189+
f(transmute(L_BASE + li));
2190+
2191+
let vi = (si % N_COUNT) / T_COUNT;
2192+
f(transmute(V_BASE + vi));
2193+
2194+
let ti = si % T_COUNT;
2195+
if ti > 0 {
2196+
f(transmute(T_BASE + ti));
2197+
}
2198+
}
2199+
}
21562200
}
21572201

21582202
pub mod derived_property {

0 commit comments

Comments
 (0)