Skip to content

Commit 401c655

Browse files
IncheonkirinIncheonkirin
authored
Add Hangul composition char filter (#16242)
1 parent 955ed9e commit 401c655

6 files changed

Lines changed: 438 additions & 0 deletions

File tree

‎lucene/CHANGES.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,9 @@ API Changes
100100

101101
New Features
102102
---------------------
103+
* GITHUB#16241: Add HangulCompositionCharFilter to analysis-nori for opt-in composition of
104+
modern Hangul conjoining jamo before KoreanTokenizer. (Mingi Jeong)
105+
103106
* GITHUB#15505: Upgrade snowball to 2d2e312df56f2ede014a4ffb3e91e6dea43c24be. New stemmer: PolishStemmer (and
104107
PolishSnowballAnalyzer in the stempel package) (Justas Sakalauskas, Dawid Weiss)
105108

‎lucene/analysis/nori/src/java/module-info.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
exports org.apache.lucene.analysis.ko.dict;
2525
exports org.apache.lucene.analysis.ko.tokenattributes;
2626

27+
provides org.apache.lucene.analysis.CharFilterFactory with
28+
org.apache.lucene.analysis.ko.HangulCompositionCharFilterFactory;
2729
provides org.apache.lucene.analysis.TokenizerFactory with
2830
org.apache.lucene.analysis.ko.KoreanTokenizerFactory;
2931
provides org.apache.lucene.analysis.TokenFilterFactory with

‎lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/HangulCompositionCharFilter.java‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.lucene.analysis.ko;
18+
19+
import java.io.IOException;
20+
import java.io.Reader;
21+
import org.apache.lucene.analysis.charfilter.BaseCharFilter;
22+
23+
/**
24+
* A {@link org.apache.lucene.analysis.CharFilter} that composes modern Hangul conjoining jamo into
25+
* precomposed Hangul syllables.
26+
*
27+
* <p>This filter only handles canonical modern Hangul syllable sequences:
28+
*
29+
* <ul>
30+
* <li>leading consonants U+1100..U+1112
31+
* <li>vowels U+1161..U+1175
32+
* <li>optional trailing consonants U+11A8..U+11C2
33+
* </ul>
34+
*
35+
* <p>Compatibility jamo and archaic jamo are left unchanged. This is intended as an opt-in
36+
* normalization step before {@link KoreanTokenizer}, whose dictionary is built for precomposed
37+
* Hangul syllables.
38+
*
39+
* <p>This filter composes only full conjoining-jamo sequences (L, V, optional T). A precomposed LV
40+
* syllable followed by a conjoining trailing consonant (e.g., U+D558 하 + U+11AB), which full NFC
41+
* would compose into 한, is intentionally left unchanged; fully decomposed (NFD) input never
42+
* contains that shape.
43+
*/
44+
public final class HangulCompositionCharFilter extends BaseCharFilter {
45+
private static final int S_BASE = 0xAC00;
46+
private static final int L_BASE = 0x1100;
47+
private static final int V_BASE = 0x1161;
48+
private static final int T_BASE = 0x11A7;
49+
50+
private static final int L_COUNT = 19;
51+
private static final int V_COUNT = 21;
52+
private static final int T_COUNT = 28;
53+
private static final int N_COUNT = V_COUNT * T_COUNT;
54+
55+
private int pushbackChar = -1;
56+
private int outputOffset = 0;
57+
58+
/** Default constructor that takes a {@link Reader}. */
59+
public HangulCompositionCharFilter(Reader input) {
60+
super(input);
61+
}
62+
63+
@Override
64+
public int read() throws IOException {
65+
int ch = nextInputChar();
66+
if (ch == -1) {
67+
return -1;
68+
}
69+
70+
int lIndex = leadingIndex(ch);
71+
if (lIndex < 0) {
72+
return emit(ch);
73+
}
74+
75+
int v = nextInputChar();
76+
if (v == -1) {
77+
return emit(ch);
78+
}
79+
80+
int vIndex = vowelIndex(v);
81+
if (vIndex < 0) {
82+
pushBack(v);
83+
return emit(ch);
84+
}
85+
86+
int consumedChars = 2;
87+
int tIndex = 0;
88+
int t = nextInputChar();
89+
if (t != -1) {
90+
tIndex = trailingIndex(t);
91+
if (tIndex > 0) {
92+
consumedChars = 3;
93+
} else {
94+
pushBack(t);
95+
tIndex = 0;
96+
}
97+
}
98+
99+
int syllable = S_BASE + (lIndex * N_COUNT) + (vIndex * T_COUNT) + tIndex;
100+
int cumulativeDiff = getLastCumulativeDiff() + consumedChars - 1;
101+
addOffCorrectMap(outputOffset + 1, cumulativeDiff);
102+
return emit(syllable);
103+
}
104+
105+
@Override
106+
public int read(char[] cbuf, int off, int len) throws IOException {
107+
int numRead = 0;
108+
for (int i = off; i < off + len; i++) {
109+
int ch = read();
110+
if (ch == -1) {
111+
break;
112+
}
113+
cbuf[i] = (char) ch;
114+
numRead++;
115+
}
116+
return numRead == 0 ? -1 : numRead;
117+
}
118+
119+
private int nextInputChar() throws IOException {
120+
if (pushbackChar != -1) {
121+
int ch = pushbackChar;
122+
pushbackChar = -1;
123+
return ch;
124+
}
125+
return input.read();
126+
}
127+
128+
private void pushBack(int ch) {
129+
assert pushbackChar == -1;
130+
pushbackChar = ch;
131+
}
132+
133+
private int emit(int ch) {
134+
outputOffset++;
135+
return ch;
136+
}
137+
138+
private static int leadingIndex(int ch) {
139+
int index = ch - L_BASE;
140+
return index >= 0 && index < L_COUNT ? index : -1;
141+
}
142+
143+
private static int vowelIndex(int ch) {
144+
int index = ch - V_BASE;
145+
return index >= 0 && index < V_COUNT ? index : -1;
146+
}
147+
148+
private static int trailingIndex(int ch) {
149+
int index = ch - T_BASE;
150+
return index > 0 && index < T_COUNT ? index : -1;
151+
}
152+
}

‎lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/HangulCompositionCharFilterFactory.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.lucene.analysis.ko;
18+
19+
import java.io.Reader;
20+
import java.util.Map;
21+
import org.apache.lucene.analysis.CharFilterFactory;
22+
23+
/**
24+
* Factory for {@link HangulCompositionCharFilter}.
25+
*
26+
* @lucene.spi {@value #NAME}
27+
*/
28+
public class HangulCompositionCharFilterFactory extends CharFilterFactory {
29+
30+
/** SPI name */
31+
public static final String NAME = "hangulComposition";
32+
33+
/** Creates a new HangulCompositionCharFilterFactory */
34+
public HangulCompositionCharFilterFactory(Map<String, String> args) {
35+
super(args);
36+
if (!args.isEmpty()) {
37+
throw new IllegalArgumentException("Unknown parameters: " + args);
38+
}
39+
}
40+
41+
/** Default ctor for compatibility with SPI */
42+
public HangulCompositionCharFilterFactory() {
43+
throw defaultCtorException();
44+
}
45+
46+
@Override
47+
public Reader create(Reader input) {
48+
return new HangulCompositionCharFilter(input);
49+
}
50+
51+
@Override
52+
public Reader normalize(Reader input) {
53+
return create(input);
54+
}
55+
}

‎lucene/analysis/nori/src/resources/META-INF/services/org.apache.lucene.analysis.CharFilterFactory‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
org.apache.lucene.analysis.ko.HangulCompositionCharFilterFactory

0 commit comments

Comments
 (0)