Natural Language Processing

import smile.util.Strings;
import smile.nlp.tokenizer.*;
import smile.nlp.stemmer.PorterStemmer;
import smile.nlp.pos.*;

String raw = "Machine learning is a subfield of AI. Deep learning uses neural nets.";
String text = Strings.normalize(raw);

var paragraphs = SimpleParagraphSplitter.getInstance();
var sentences = SimpleSentenceSplitter.getInstance();
var tokenizer = new SimpleTokenizer();
var tagger = HMMPOSTagger.getDefault();
var stemmer = new PorterStemmer();

for (String para : paragraphs.split(text)) {
    for (String sent : sentences.split(para)) {
        String[] tokens = tokenizer.split(sent);
        PennTreebankPOS[] tags = tagger.tag(tokens);
        for (int i = 0; i < tokens.length; i++) {
            if (tags[i].open) {
                System.out.printf("%-16s %-6s %s%n",
                    tokens[i], tags[i], stemmer.stem(tokens[i].toLowerCase()));
            }
        }
    }
}

Machine          NN     machin
learning         NN     learn
...
      

import smile.nlp._
import smile.nlp.pos.PennTreebankPOS

val raw = "Machine learning is a subfield of AI. Deep learning uses neural nets."
for (sent <- raw.normalize.sentences) {
  val words = sent.words("none")
  val tags = postag(words)
  words.zip(tags).foreach { case (w, t) =>
    if (t.open) println(f"$w%-16s $t%-6s ${porter.stem(w.toLowerCase)}")
  }
}

Machine          NN     machin
learning         NN     learn
...
      

import smile.nlp.*

val raw = "Machine learning is a subfield of AI. Deep learning uses neural nets."
for (sent in raw.normalize().sentences()) {
    val words = sent.words("none")
    val tags = postag(words)
    for (i in words.indices) {
        if (tags[i].open) {
            println("%-16s %-6s %s".format(
                words[i], tags[i], porter(words[i].lowercase())))
        }
    }
}

Machine          NN     machin
learning         NN     learn
...
      

import smile.nlp.Text;

Text doc = Text.of("Machine learning is a subfield of AI.");
Text titled = Text.of("Overview", "Machine learning is a subfield of AI.");
System.out.println(doc.content());
System.out.println(titled.title());

Machine learning is a subfield of AI.
Overview
      

import smile.nlp.Text

val doc = Text.of("Machine learning is a subfield of AI.")
val titled = Text.of("Overview", "Machine learning is a subfield of AI.")
println(doc.content)
println(titled.title)

Machine learning is a subfield of AI.
Overview
      

import smile.nlp.Text

val doc = Text.of("Machine learning is a subfield of AI.")
val titled = Text.of("Overview", "Machine learning is a subfield of AI.")
println(doc.content())
println(titled.title())

Machine learning is a subfield of AI.
Overview
      

import smile.nlp.SimpleCorpus;

SimpleCorpus corpus = new SimpleCorpus();
corpus.add(corpus.doc("SMILE is a fast machine learning library."));
corpus.add(corpus.doc("Deep learning uses neural networks."));
System.out.println(corpus.docCount());
System.out.println(corpus.count("learning"));

2
2
      

import smile.nlp._

val c = corpus(Seq(
  "SMILE is a fast machine learning library.",
  "Deep learning uses neural networks."
))
println(c.docCount)
println(c.count("learning"))

2
2
      

import smile.nlp.*

val c = corpus(listOf(
    "SMILE is a fast machine learning library.",
    "Deep learning uses neural networks."
))
println(c.docCount())
println(c.count("learning"))

2
2
      

import smile.util.Strings;

String clean = Strings.normalize("\u201CHello,\u201D   world\u2019s\u2026");
System.out.println(clean);

"Hello," world's...
      

import smile.nlp._

val clean = "\u201CHello,\u201D   world\u2019s\u2026".normalize
println(clean)

"Hello," world's...
      

import smile.nlp.*

val clean = "\u201CHello,\u201D   world\u2019s\u2026".normalize()
println(clean)

"Hello," world's...
      

import smile.nlp.tokenizer.*;

String text = "Dr. Smith lives in the U.S. He works at NASA.";
String[] sents = SimpleSentenceSplitter.getInstance().split(text);
Tokenizer tok = new SimpleTokenizer();
for (String s : sents) {
    System.out.println(String.join(" | ", tok.split(s)));
}

Dr. | Smith | lives | in | the | U.S. | .
He | works | at | NASA | .
      

import smile.nlp._

val text = "Dr. Smith lives in the U.S. He works at NASA."
text.sentences.foreach { s =>
  println(s.words("none").mkString(" | "))
}

Dr. | Smith | lives | in | the | U.S. | .
He | works | at | NASA | .
      

import smile.nlp.*

val text = "Dr. Smith lives in the U.S. He works at NASA."
for (s in text.sentences()) {
    println(s.words("none").joinToString(" | "))
}

Dr. | Smith | lives | in | the | U.S. | .
He | works | at | NASA | .
      

import java.util.Arrays;
import smile.nlp.tokenizer.*;

String s = "They couldn't have known.";
Tokenizer simple = new SimpleTokenizer();
Tokenizer ptb = PennTreebankTokenizer.getInstance();
System.out.println(Arrays.toString(simple.split(s)));
System.out.println(Arrays.toString(ptb.split(s)));

[They, couldn't, have, known, .]
[They, could, n't, have, known, .]
      

import smile.nlp.tokenizer._

val s = "They couldn't have known."
println(new SimpleTokenizer().split(s).mkString("[", ", ", "]"))
println(PennTreebankTokenizer.getInstance.split(s).mkString("[", ", ", "]"))

[They, couldn't, have, known, .]
[They, could, n't, have, known, .]
      

import smile.nlp.tokenizer.*

val s = "They couldn't have known."
println(SimpleTokenizer().split(s).contentToString())
println(PennTreebankTokenizer.getInstance().split(s).contentToString())

[They, couldn't, have, known, .]
[They, could, n't, have, known, .]
      

import smile.nlp.stemmer.*;

PorterStemmer porter = new PorterStemmer();
LancasterStemmer lancaster = new LancasterStemmer();
System.out.println(porter.stem("generalization"));
System.out.println(lancaster.stem("presumably"));

gener
presum
      

import smile.nlp._

println(porter.stem("generalization"))
println(lancaster.stem("presumably"))

gener
presum
      

import smile.nlp.*

println(porter("generalization"))
println(lancaster("presumably"))

gener
presum
      

import smile.nlp.stemmer.PorterStemmer;

PorterStemmer porter = new PorterStemmer();
System.out.println(porter.stripPluralParticiple("cats"));
System.out.println(porter.stripPluralParticiple("running"));
System.out.println(porter.stripPluralParticiple("was"));
System.out.println(porter.stem("generalization"));

cat
run
was
gener
      

import smile.nlp.stemmer.PorterStemmer

val porter = new PorterStemmer
println(porter.stripPluralParticiple("cats"))
println(porter.stripPluralParticiple("running"))
println(porter.stripPluralParticiple("was"))
println(porter.stem("generalization"))

cat
run
was
gener
      

import smile.nlp.stemmer.PorterStemmer

val porter = PorterStemmer()
println(porter.stripPluralParticiple("cats"))
println(porter.stripPluralParticiple("running"))
println(porter.stripPluralParticiple("was"))
println(porter.stem("generalization"))

cat
run
was
gener
      

import smile.nlp.stemmer.LancasterStemmer;

LancasterStemmer lancaster = new LancasterStemmer(true);
System.out.println(lancaster.stem("microprocessor"));
System.out.println(lancaster.stem("kilowatt"));
System.out.println(lancaster.stem("pseudoscience"));

process
wat
scy
      

import smile.nlp.stemmer.LancasterStemmer

val lancaster = new LancasterStemmer(true)
println(lancaster.stem("microprocessor"))
println(lancaster.stem("kilowatt"))
println(lancaster.stem("pseudoscience"))

process
wat
scy
      

import smile.nlp.stemmer.LancasterStemmer

val lancaster = LancasterStemmer(true)
println(lancaster.stem("microprocessor"))
println(lancaster.stem("kilowatt"))
println(lancaster.stem("pseudoscience"))

process
wat
scy
      

import smile.nlp.pos.*;

HMMPOSTagger tagger = HMMPOSTagger.getDefault();
String[] tokens = {"The", "cat", "sat", "on", "the", "mat", "."};
PennTreebankPOS[] tags = tagger.tag(tokens);
for (int i = 0; i < tokens.length; i++) {
    System.out.printf("%-12s %s%n", tokens[i], tags[i]);
}

The          DT
cat          NN
sat          VBD
on           IN
the          DT
mat          NN
.            .
      

import smile.nlp._

val tokens = Array("The", "cat", "sat", "on", "the", "mat", ".")
tokens.zip(postag(tokens)).foreach { case (w, t) =>
  println(f"$w%-12s $t")
}

The          DT
cat          NN
sat          VBD
on           IN
the          DT
mat          NN
.            .
      

import smile.nlp.*

val tokens = arrayOf("The", "cat", "sat", "on", "the", "mat", ".")
val tags = postag(tokens)
for (i in tokens.indices) {
    println("%-12s %s".format(tokens[i], tags[i]))
}

The          DT
cat          NN
sat          VBD
on           IN
the          DT
mat          NN
.            .
      

import smile.nlp.dictionary.EnglishStopWords;

EnglishStopWords stop = EnglishStopWords.DEFAULT;
System.out.println(stop.contains("the"));
System.out.println(stop.contains("machine"));

true
false
      

import smile.nlp.dictionary.EnglishStopWords

val stop = EnglishStopWords.DEFAULT
println(stop.contains("the"))
println(stop.contains("machine"))

true
false
      

import smile.nlp.dictionary.EnglishStopWords

val stop = EnglishStopWords.DEFAULT
println(stop.contains("the"))
println(stop.contains("machine"))

true
false
      

import smile.nlp.dictionary.EnglishPunctuations;

EnglishPunctuations punct = EnglishPunctuations.getInstance();
System.out.println(punct.contains("."));
System.out.println(punct.contains("NLP"));

true
false
      

import smile.nlp.dictionary.EnglishPunctuations

val punct = EnglishPunctuations.getInstance
println(punct.contains("."))
println(punct.contains("NLP"))

true
false
      

import smile.nlp.dictionary.EnglishPunctuations

val punct = EnglishPunctuations.getInstance()
println(punct.contains("."))
println(punct.contains("NLP"))

true
false
      

import smile.nlp.dictionary.EnglishDictionary;

System.out.println(EnglishDictionary.CONCISE.contains("serendipity"));
System.out.println(EnglishDictionary.CONCISE.contains("xyzzy"));

true
false
      

import smile.nlp.dictionary.EnglishDictionary

println(EnglishDictionary.CONCISE.contains("serendipity"))
println(EnglishDictionary.CONCISE.contains("xyzzy"))

true
false
      

import smile.nlp.dictionary.EnglishDictionary

println(EnglishDictionary.CONCISE.contains("serendipity"))
println(EnglishDictionary.CONCISE.contains("xyzzy"))

true
false
      

import smile.nlp.SimpleCorpus;

SimpleCorpus corpus = new SimpleCorpus();
corpus.add(corpus.doc("Machine learning is part of artificial intelligence."));
corpus.add(corpus.doc("Deep learning is a type of machine learning."));
corpus.add(corpus.doc("Neural networks power deep learning systems."));

System.out.println("--- top-k ---");
corpus.bigrams(5, 1).forEach(b ->
    System.out.println(b.w1() + " " + b.w2() + "  score=" + b.score()));

System.out.println("--- p-value ---");
corpus.bigrams(0.01, 1).forEach(b ->
    System.out.println(b.w1() + " " + b.w2() + "  score=" + b.score()));

--- top-k ---
machine learning  score=...
--- p-value ---
...
      

import smile.nlp._

val docs = Seq(
  "Machine learning is part of artificial intelligence.",
  "Deep learning is a type of machine learning.",
  "Neural networks power deep learning systems.")
println("--- top-k ---")
bigram(5, 1, docs: _*).foreach(b => println(s"${b.w1} ${b.w2}  score=${b.score}"))
println("--- p-value ---")
bigram(0.01, 1, docs: _*).foreach(b => println(s"${b.w1} ${b.w2}  score=${b.score}"))

--- top-k ---
machine learning  score=...
--- p-value ---
...
      

import smile.nlp.*

val docs = listOf(
    "Machine learning is part of artificial intelligence.",
    "Deep learning is a type of machine learning.",
    "Neural networks power deep learning systems."
)
println("--- top-k ---")
for (b in bigrams(5, 1, docs)) {
    println("${b.w1()} ${b.w2()}  score=${b.score()}")
}
println("--- p-value ---")
for (b in bigrams(0.01, 1, docs)) {
    println("${b.w1()} ${b.w2()}  score=${b.score()}")
}

--- top-k ---
machine learning  score=...
--- p-value ---
...
      

import java.util.List;
import smile.nlp.NGram;
import smile.nlp.tokenizer.*;

String[] docs = {
    "Machine learning is part of artificial intelligence.",
    "Deep learning is a type of machine learning."
};
var splitter = SimpleSentenceSplitter.getInstance();
var tokenizer = new SimpleTokenizer();
List<String[]> sentences = java.util.Arrays.stream(docs)
    .flatMap(d -> java.util.Arrays.stream(splitter.split(d)))
    .map(tokenizer::split)
    .toList();

NGram[][] result = NGram.apriori(sentences, 3, 2);
for (NGram g : result[2]) {
    System.out.println(String.join(" ", g.words()) + "  count=" + g.count());
}

machin learn  count=...
      

import smile.nlp._

val result = ngram(3, 2,
  "Machine learning is part of artificial intelligence.",
  "Deep learning is a type of machine learning.")
result(2).foreach(g => println(g.words.mkString(" ") + "  count=" + g.count))

machin learn  count=...
      

import smile.nlp.*

val result = ngram(3, 2, listOf(
    "Machine learning is part of artificial intelligence.",
    "Deep learning is a type of machine learning."
))
for (g in result[2]) {
    println(g.words().joinToString(" ") + "  count=" + g.count())
}

machin learn  count=...
      

import smile.nlp.Text;

String body = "Machine learning is a field of artificial intelligence. "
    + "Deep learning is a subfield of machine learning that uses neural networks. "
    + "Neural networks learn hierarchical representations from data.";
Text.of(body).keywords(5).forEach(k ->
    System.out.println(String.join(" ", k.words()) + "  count=" + k.count()));

machin learn  count=...
deep learn  count=...
      

import smile.nlp._

val body =
  """Machine learning is a field of artificial intelligence.
     Deep learning is a subfield of machine learning that uses neural networks.
     Neural networks learn hierarchical representations from data."""
body.keywords(5).foreach(k =>
  println(k.words.mkString(" ") + "  count=" + k.count))

machin learn  count=...
deep learn  count=...
      

import smile.nlp.*

val body = """
    Machine learning is a field of artificial intelligence.
    Deep learning is a subfield of machine learning that uses neural networks.
    Neural networks learn hierarchical representations from data.
    """.trimIndent()
for (k in body.keywords(5)) {
    println(k.words().joinToString(" ") + "  count=" + k.count())
}

machin learn  count=...
deep learn  count=...
      

import smile.nlp.Text;
import smile.nlp.SimpleCorpus;
import smile.nlp.relevance.*;

SimpleCorpus corpus = new SimpleCorpus();
corpus.add(corpus.doc(Text.of("Intro", "SMILE is a fast machine learning library.")));
corpus.add(corpus.doc(Text.of("DL", "Deep learning uses neural networks.")));
corpus.add(corpus.doc(Text.of("IR", "BM25 ranks documents for information retrieval.")));

RelevanceRanker bm25 = new BM25();
corpus.search(bm25, new String[]{"machine", "learning"})
    .forEachRemaining(r ->
        System.out.printf("%.4f  %s%n", r.score(), r.text().title()));

0.xxxx  Intro
...
      

import smile.nlp._
import smile.nlp.Text
import smile.nlp.relevance.BM25
import scala.jdk.CollectionConverters._

val c = new SimpleCorpus
c.add(c.doc(Text.of("Intro", "SMILE is a fast machine learning library.")))
c.add(c.doc(Text.of("DL", "Deep learning uses neural networks.")))
c.add(c.doc(Text.of("IR", "BM25 ranks documents for information retrieval.")))
c.search(new BM25, Array("machine", "learning")).asScala.foreach { r =>
  println(f"${r.score}%.4f  ${r.text.title}")
}

0.xxxx  Intro
...
      

import smile.nlp.*
import smile.nlp.relevance.BM25

val c = SimpleCorpus()
c.add(c.doc(Text.of("Intro", "SMILE is a fast machine learning library.")))
c.add(c.doc(Text.of("DL", "Deep learning uses neural networks.")))
c.add(c.doc(Text.of("IR", "BM25 ranks documents for information retrieval.")))
c.search(BM25(), arrayOf("machine", "learning")).forEachRemaining { r ->
    println("%.4f  %s".format(r.score(), r.text().title()))
}

0.xxxx  Intro
...