topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2026-04-24 00:04:26 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)[build] Use Spotless to customize the code formatting (#1750)

Browse Source
This commit is contained in:
lexluo09
2024-10-04 00:05:04 +08:00
committed by GitHub
parent 44d1cde34f
commit 71a9954be5
521 changed files with 7811 additions and 13046 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java
View File

@@ -24,13 +24,12 @@ public class LoadRemoveService {
}
List<String> resultList = new ArrayList<>(value);
if (StringUtils.isNotBlank(mapperRemoveNaturePrefix)) {
resultList.removeIf(
nature -> {
if (Objects.isNull(nature)) {
return false;
}
return nature.startsWith(mapperRemoveNaturePrefix);
});
resultList.removeIf(nature -> {
if (Objects.isNull(nature)) {
return false;
}
return nature.startsWith(mapperRemoveNaturePrefix);
});
}
return resultList;
}

15
common/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java
View File

@@ -253,19 +253,8 @@ public abstract class BaseNode<V> implements Comparable<BaseNode> {
@Override
public String toString() {
return "BaseNode{"
+ "child="
+ Arrays.toString(child)
+ ", status="
+ status
+ ", c="
+ c
+ ", value="
+ value
+ ", prefix='"
+ prefix
+ '\''
+ '}';
return "BaseNode{" + "child=" + Arrays.toString(child) + ", status=" + status + ", c=" + c
+ ", value=" + value + ", prefix='" + prefix + '\'' + '}';
}
public void walkNode(Set<Map.Entry<String, V>> entrySet) {

36
common/src/main/java/com/hankcs/hanlp/dictionary/CoreDictionary.java
View File

@@ -34,13 +34,8 @@ public class CoreDictionary {
if (!load(PATH)) {
throw new IllegalArgumentException("核心词典" + PATH + "加载失败");
} else {
Predefine.logger.info(
PATH
+ "加载成功，"
+ trie.size()
+ "个词条，耗时"
+ (System.currentTimeMillis() - start)
+ "ms");
Predefine.logger.info(PATH + "加载成功，" + trie.size() + "个词条，耗时"
+ (System.currentTimeMillis() - start) + "ms");
}
}
@@ -77,22 +72,14 @@ public class CoreDictionary {
map.put(param[0], attribute);
totalFrequency += attribute.totalFrequency;
}
Predefine.logger.info(
"核心词典读入词条"
+ map.size()
+ " 全部频次"
+ totalFrequency
+ "，耗时"
+ (System.currentTimeMillis() - start)
+ "ms");
Predefine.logger.info("核心词典读入词条" + map.size() + " 全部频次" + totalFrequency + "，耗时"
+ (System.currentTimeMillis() - start) + "ms");
br.close();
trie.build(map);
Predefine.logger.info("核心词典加载成功:" + trie.size() + "个词条，下面将写入缓存……");
try {
DataOutputStream out =
new DataOutputStream(
new BufferedOutputStream(
IOUtil.newOutputStream(path + Predefine.BIN_EXT)));
DataOutputStream out = new DataOutputStream(
new BufferedOutputStream(IOUtil.newOutputStream(path + Predefine.BIN_EXT)));
Collection<Attribute> attributeList = map.values();
out.writeInt(attributeList.size());
for (Attribute attribute : attributeList) {
@@ -278,11 +265,8 @@ public class CoreDictionary {
}
return attribute;
} catch (Exception e) {
Predefine.logger.warning(
"使用字符串"
+ natureWithFrequency
+ "创建词条属性失败！"
+ TextUtility.exceptionToString(e));
Predefine.logger.warning("使用字符串" + natureWithFrequency + "创建词条属性失败！"
+ TextUtility.exceptionToString(e));
return null;
}
}
@@ -409,9 +393,7 @@ public class CoreDictionary {
if (originals == null || originals.length == 0) {
return null;
}
return Arrays.stream(originals)
.filter(o -> o != null)
.distinct()
return Arrays.stream(originals).filter(o -> o != null).distinct()
.collect(Collectors.toList());
}
}

55
common/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java
View File

@@ -47,8 +47,7 @@ public abstract class WordBasedSegment extends Segment {
}
vertex = (Vertex) var1.next();
} while (!vertex.realWord.equals("－－")
&& !vertex.realWord.equals("—")
} while (!vertex.realWord.equals("－－") && !vertex.realWord.equals("—")
&& !vertex.realWord.equals("-"));
vertex.confirmNature(Nature.w);
@@ -66,8 +65,7 @@ public abstract class WordBasedSegment extends Segment {
if (currentNature == Nature.nx
&& (next.hasNature(Nature.q) || next.hasNature(Nature.n))) {
String[] param = current.realWord.split("-", 1);
if (param.length == 2
&& TextUtility.isAllNum(param[0])
if (param.length == 2 && TextUtility.isAllNum(param[0])
&& TextUtility.isAllNum(param[1])) {
current = current.copy();
current.realWord = param[0];
@@ -112,10 +110,8 @@ public abstract class WordBasedSegment extends Segment {
current.confirmNature(Nature.m, true);
} else if (current.realWord.length() > 1) {
char last = current.realWord.charAt(current.realWord.length() - 1);
current =
Vertex.newNumberInstance(
current.realWord.substring(
0, current.realWord.length() - 1));
current = Vertex.newNumberInstance(
current.realWord.substring(0, current.realWord.length() - 1));
listIterator.previous();
listIterator.previous();
listIterator.set(current);
@@ -162,9 +158,7 @@ public abstract class WordBasedSegment extends Segment {
charTypeArray[i] = CharType.get(c);
if (c == '.' && i < charArray.length - 1 && CharType.get(charArray[i + 1]) == 9) {
charTypeArray[i] = 9;
} else if (c == '.'
&& i < charArray.length - 1
&& charArray[i + 1] >= '0'
} else if (c == '.' && i < charArray.length - 1 && charArray[i + 1] >= '0'
&& charArray[i + 1] <= '9') {
charTypeArray[i] = 5;
} else if (charTypeArray[i] == 8) {
@@ -227,7 +221,7 @@ public abstract class WordBasedSegment extends Segment {
while (listIterator.hasNext()) {
next = (Vertex) listIterator.next();
if (!TextUtility.isAllNum(current.realWord)
&& !TextUtility.isAllChineseNum(current.realWord)
&& !TextUtility.isAllChineseNum(current.realWord)
|| !TextUtility.isAllNum(next.realWord)
&& !TextUtility.isAllChineseNum(next.realWord)) {
current = next;
@@ -252,21 +246,16 @@ public abstract class WordBasedSegment extends Segment {
DoubleArrayTrie.Searcher searcher = CoreDictionary.trie.getSearcher(charArray, 0);
while (searcher.next()) {
wordNetStorage.add(
searcher.begin + 1,
new Vertex(
new String(charArray, searcher.begin, searcher.length),
(CoreDictionary.Attribute) searcher.value,
searcher.index));
wordNetStorage.add(searcher.begin + 1,
new Vertex(new String(charArray, searcher.begin, searcher.length),
(CoreDictionary.Attribute) searcher.value, searcher.index));
}
if (this.config.forceCustomDictionary) {
this.customDictionary.parseText(
charArray,
this.customDictionary.parseText(charArray,
new AhoCorasickDoubleArrayTrie.IHit<CoreDictionary.Attribute>() {
public void hit(int begin, int end, CoreDictionary.Attribute value) {
wordNetStorage.add(
begin + 1,
wordNetStorage.add(begin + 1,
new Vertex(new String(charArray, begin, end - begin), value));
}
});
@@ -279,11 +268,9 @@ public abstract class WordBasedSegment extends Segment {
while (i < vertexes.length) {
if (vertexes[i].isEmpty()) {
int j;
for (j = i + 1;
j < vertexes.length - 1
&& (vertexes[j].isEmpty()
|| CharType.get(charArray[j - 1]) == 11);
++j) {}
for (j = i + 1; j < vertexes.length - 1 && (vertexes[j].isEmpty()
|| CharType.get(charArray[j - 1]) == 11); ++j) {
}
wordNetStorage.add(i, Segment.quickAtomSegment(charArray, i - 1, j - 1));
i = j;
@@ -310,10 +297,8 @@ public abstract class WordBasedSegment extends Segment {
addTerms(termList, vertex, line - 1);
termMain.offset = line - 1;
if (vertex.realWord.length() > 2) {
label43:
for (int currentLine = line;
currentLine < line + vertex.realWord.length();
++currentLine) {
label43: for (int currentLine = line; currentLine < line
+ vertex.realWord.length(); ++currentLine) {
Iterator iterator = wordNetAll.descendingIterator(currentLine);
while (true) {
@@ -327,8 +312,8 @@ public abstract class WordBasedSegment extends Segment {
&& smallVertex.realWord.length() < this.config.indexMode);
if (smallVertex != vertex
&& currentLine + smallVertex.realWord.length()
<= line + vertex.realWord.length()) {
&& currentLine + smallVertex.realWord.length() <= line
+ vertex.realWord.length()) {
listIterator.add(smallVertex);
// Term termSub = convert(smallVertex);
// termSub.offset = currentLine - 1;
@@ -346,8 +331,8 @@ public abstract class WordBasedSegment extends Segment {
}
protected static void speechTagging(List<Vertex> vertexList) {
Viterbi.compute(
vertexList, CoreDictionaryTransformMatrixDictionary.transformMatrixDictionary);
Viterbi.compute(vertexList,
CoreDictionaryTransformMatrixDictionary.transformMatrixDictionary);
}
protected void addTerms(List<Term> terms, Vertex vertex, int offset) {

20
common/src/main/java/com/hankcs/hanlp/seg/common/Term.java
View File

@@ -42,19 +42,13 @@ public class Term {
}
// todo opt
/*
String wordOri = word.toLowerCase();
CoreDictionary.Attribute attribute = getDynamicCustomDictionary().get(wordOri);
if (attribute == null) {
attribute = CoreDictionary.get(wordOri);
if (attribute == null) {
attribute = CustomDictionary.get(wordOri);
}
}
if (attribute != null && nature != null && attribute.hasNature(nature)) {
return attribute.getNatureFrequency(nature);
}
return attribute == null ? 0 : attribute.totalFrequency;
*/
* String wordOri = word.toLowerCase(); CoreDictionary.Attribute attribute =
* getDynamicCustomDictionary().get(wordOri); if (attribute == null) { attribute =
* CoreDictionary.get(wordOri); if (attribute == null) { attribute =
* CustomDictionary.get(wordOri); } } if (attribute != null && nature != null &&
* attribute.hasNature(nature)) { return attribute.getNatureFrequency(nature); } return
* attribute == null ? 0 : attribute.totalFrequency;
*/
return 0;
}