Large Language Models

import smile.llm.Message;
import smile.llm.ChatCompletion;

Message[] dialog = {
    Message.system("You are a helpful assistant."),
    Message.user("What is the capital of France?")
};

ChatCompletion reply = llama.chat(dialog, 256, 0.6, 0.9, false, 0L, null);
System.out.println(reply.content());   // "The capital of France is Paris."
System.out.println(reply.reason());    // FinishReason.stop

import smile.llm.Message;
import smile.llm.Role;
import smile.llm.TextPart;
import smile.llm.ImageUrlPart;
import java.util.List;

// Multimodal turn: text interleaved with an image
Message turn = new Message(Role.user, List.of(
        new TextPart("What is in this picture?"),
        new ImageUrlPart("https://example.com/cat.jpg")));

// Assistant tool call + tool result
Message call   = new Message(Role.assistant, List.of(),
        List.of(toolCall), null, null);
Message result = new Message(Role.tool, "{\"temp\": 21}",
        null, toolCall.id(), null);

import smile.llm.tokenizer.Tiktoken;

Tiktoken tok = new Tiktoken(
        pattern,           // regex splitting pattern
        specialTokens,     // map of special-token string → rank
        ranks,             // merged BPE vocabulary (bytes → rank)
        bosId, eosId       // BOS / EOS token IDs
);

int[] ids = tok.encode("Hello, world!", true, true);  // BOS+EOS
String text = tok.decode(ids);
int vocab = tok.size();

import smile.deep.tensor.Tensor;
import smile.llm.transformer.PositionalEncoding;
import smile.llm.transformer.RotaryPositionalEncoding;

PositionalEncoding pe = new PositionalEncoding(512, 2048);
Tensor out = pe.forward(embeddingTensor);   // adds positional signal

RotaryPositionalEncoding rope =
        new RotaryPositionalEncoding(headDim, maxSeqLen);

import smile.llm.model.llama.Llama;

// Loads params.json / config.json + checkpoint weights
Llama llama = Llama.build(
        "model/Meta-Llama-3-8B-Instruct",
        "model/Meta-Llama-3-8B-Instruct/tokenizer.model",
        /*maxBatchSize=*/ 4,
        /*maxSeqLen=*/    2048,
        /*deviceId=*/     (byte) 0           // CUDA:0; -1 for CPU
);

import smile.llm.ChatCompletion;
import smile.llm.Message;

// Raw token IDs
int[] prompt = llama.tokenizer.encode("Once upon a time", true, false);
ChatCompletion result = llama.generate(
        prompt, /*maxGenLen=*/ 200, /*temperature=*/ 0.6,
        /*topp=*/ 0.9, /*logprobs=*/ false, /*seed=*/ 42L,
        /*listener=*/ null);
System.out.println(result.content());

// Dialog format
ChatCompletion reply = llama.chat(
        new Message[]{
            Message.system("Be concise."),
            Message.user("Explain RoPE in one sentence.")
        },
        /*maxGenLen=*/ 128, /*temperature=*/ 0.7, /*topp=*/ 0.9,
        /*logprobs=*/ false, /*seed=*/ 0L, /*listener=*/ null);
System.out.println(reply.content());

import smile.llm.GenerationListener;

GenerationListener listener = new GenerationListener() {
    @Override public void onText(String chunk) {
        System.out.print(chunk);
    }
};
int[] prompt = llama.tokenizer.encode("Tell me a joke", true, false);
llama.generate(prompt, 200, 0.8, 0.95, false, 0L, listener);

import smile.llm.model.qwen.Qwen;
import smile.llm.ChatCompletion;
import smile.llm.Message;

Qwen qwen = Qwen.build(
        "model/Qwen3.5-8B",
        /*maxBatchSize=*/ 4,
        /*maxSeqLen=*/    4096,   // <= 0 uses the config value
        /*deviceId=*/     (byte) 0
);

ChatCompletion out = qwen.complete(
        "Write a haiku about the sea.", 64, 0.7, 0.9, false, 0L, null);

ChatCompletion reply = qwen.chat(
        new Message[]{
            Message.system("You are a helpful assistant."),
            Message.user("What are the key differences between Java and Kotlin?")
        },
        /*maxGenLen=*/ 512, /*temperature=*/ 0.7, /*topp=*/ 0.9,
        /*logprobs=*/ false, /*seed=*/ 0L, /*listener=*/ null);
System.out.println(reply.content());

qwen.setSpeculativeEnabled(true);    // native MTP draft/verify
qwen.setPrefixReplayEnabled(true);   // radix prefix + DeltaNet restore

import smile.llm.engine.InferenceEngine;
import smile.llm.engine.GenerationRequest;
import smile.llm.engine.GenerationHandle;
import smile.llm.ChatCompletion;

try (var engine = new InferenceEngine(llama, /*maxInFlight=*/ 8)) {
    GenerationHandle handle = engine.submit(
            GenerationRequest.ofTokens(
                    prompt, 256, 0.6, 0.9, false, 0L, listener));
    ChatCompletion reply = handle.future().join();
}

import smile.llm.cache.KvCachePool;
import smile.torch.ScalarType;

KvCachePool.StaticKvBudget budget =
        KvCachePool.computeStaticKvBudget(...);
KvCachePool pool = KvCachePool.allocate(
        layout, device, ScalarType.BFloat16, budget);
pool.setPrefixReuseEnabled(true);   // radix match/insert

import smile.llm.attention.AttentionBackends;
import smile.llm.attention.AttentionBackend;

AttentionBackends.install(AttentionBackend.FLASHINFER);

import smile.llm.quant.QuantPolicy;

QuantPolicy.Resolved resolved =
        QuantPolicy.resolve(checkpointDir, device, backendOverride);
// resolved.format()  → QuantFormat
// resolved.backend() → WeightGemmBackend
//                    (FP8 / NVFP4 primary, Marlin Ampere failover)

import smile.llm.parallel.ParallelConfig;
import smile.llm.model.qwen.Qwen;

ParallelConfig tp = ParallelConfig.tensorParallel((byte) 0, (byte) 1);
Qwen qwen = Qwen.build(dir, /*maxBatchSize=*/ 8, /*maxSeqLen=*/ 8192,
        (byte) 0, /*memFractionStatic=*/ 0.85, "bfloat16", tp);

import smile.llm.ChatOptions;
import smile.llm.ToolDefinition;
import smile.llm.FunctionDefinition;
import smile.llm.ToolChoice;

ChatOptions options = new ChatOptions(
        new ToolDefinition[]{ new ToolDefinition(new FunctionDefinition(
                "get_weather", "Get the current weather", parametersSchema)) },
        ToolChoice.AUTO,
        /*parallelToolCalls=*/ true);

int[] prompt = model.encodeChat(dialog, options);