#include #include #include #include #include #include #include #include #include "Xenith/core.hpp" #include "Xenith/token/token.hpp" #include std::string currentSystemPrompt = ""; LayerStructure_t layers[] = { {MAX_CONTEXT * EMBED_DIM, SIGMOID}, {256, SIGMOID}, {MAX_VOCAB, SIGMOID} }; std::vector buildNetInput(const std::vector& tokens, Embedder& emb) { std::vector netInput; netInput.reserve(MAX_CONTEXT * EMBED_DIM); int start = (int)tokens.size() - MAX_CONTEXT; if (start < 0) start = 0; int count = 0; for (int i = start; i < (int)tokens.size(); i++) { std::vector v = emb.get(tokens[i]); netInput.insert(netInput.end(), v.begin(), v.end()); count++; } while (count < MAX_CONTEXT) { for (int d = 0; d < EMBED_DIM; d++) netInput.push_back(0.0); count++; } return netInput; } void trainOnSequence(NeuralNetwork& nn, Tokenizer& tok, Embedder& emb, const std::string& dataset, int epochs, double lr) { std::vector allTokens = tok.textToTokens(dataset); if (allTokens.size() < 2) { std::cout << "Error: Sequence too short for training." << std::endl; return; } int numLayers = sizeof(layers) / sizeof(layers[0]); long long totalParams = 0; for (int i = 0; i < numLayers - 1; i++) { totalParams += (long long)layers[i].size * layers[i + 1].size + layers[i + 1].size; } std::string modelSizeStr; { std::stringstream ss; if (totalParams >= 1e12) ss << std::fixed << std::setprecision(1) << totalParams / 1e12 << "t"; else if (totalParams >= 1e9) ss << std::fixed << std::setprecision(1) << totalParams / 1e9 << "b"; else if (totalParams >= 1e6) ss << std::fixed << std::setprecision(1) << totalParams / 1e6 << "m"; else if (totalParams >= 1e3) ss << std::fixed << std::setprecision(1) << totalParams / 1e3 << "k"; else ss << totalParams; modelSizeStr = ss.str(); } std::string sequenceStr = ""; for (int tId : allTokens) { sequenceStr += "{" + tok.getWord(tId) + " (" + std::to_string(tId) + ")} "; } auto startTime = std::chrono::high_resolution_clock::now(); int trainSteps = 0; double stepsPerSec = 0, maxLoss = 0; std::cout << "Training logic: Next Token Prediction..." << std::endl; std::cout << "\033[s\n\n"; for (int e = 1; e <= epochs; e++) { double totalLoss = 0; for (size_t i = 1; i < allTokens.size(); i++) { std::vector context(allTokens.begin(), allTokens.begin() + i); std::vector target(MAX_VOCAB, 0.0); target[allTokens[i]] = 1.0; totalLoss += nn.train(buildNetInput(context, emb), target, lr); trainSteps++; auto currentTime = std::chrono::high_resolution_clock::now(); if (std::chrono::duration(currentTime - startTime).count() >= 0.1) { stepsPerSec = trainSteps / std::chrono::duration(currentTime - startTime).count(); trainSteps = 0; startTime = currentTime; } std::cout << "\033[u"; std::cout << "Epoch " << std::setw(4) << e << "/" << epochs << " | Token: " << std::setw(4) << i << "/" << allTokens.size() << " | Loss: " << std::fixed << std::setprecision(6) << totalLoss << " | Max Loss: " << maxLoss << "\033[K\n"; std::cout << "SPEED: " << std::setw(6) << std::fixed << std::setprecision(1) << stepsPerSec << " st/s | MODEL: " << std::setw(7) << modelSizeStr << " | CURRENT: [" << std::left << std::setw(15) << tok.getWord(allTokens[i]) << "]" << "\033[K" << std::flush; } maxLoss = totalLoss; } std::cout << "\nDone!" << std::endl; } int main() { Tokenizer tok; Embedder emb(MAX_VOCAB, EMBED_DIM); int numLayers = sizeof(layers) / sizeof(layers[0]); NeuralNetwork nn(layers, numLayers, true); while (true) { std::cout << "xentith~$ "; std::string cmdIn; std::getline(std::cin, cmdIn); if (cmdIn == "/exit") break; if (cmdIn == "/train") { int epochs; double lr; std::cout << "--- Training Setup ---\n"; std::cout << "Enter number of epochs: "; std::string epStr; std::getline(std::cin, epStr); epochs = std::stoi(epStr); std::cout << "Enter learning rate (e.g. 0.1): "; std::string lrStr; std::getline(std::cin, lrStr); lr = std::stod(lrStr); std::cout << "\n--- Example Content ---\n"; std::cout << "User: "; std::string userPart; std::getline(std::cin, userPart); std::cout << "AI: "; std::string aiPart; std::getline(std::cin, aiPart); std::string finalData = "[SYS]" + currentSystemPrompt + "[USER]" + userPart + "[AI]" + aiPart + ""; std::cout << "\nTraining logic: Pattern Recognition..." << std::endl; trainOnSequence(nn, tok, emb, finalData, epochs, lr); } else if (cmdIn == "/trainFile") { std::string content; std::cout << "Enter filename: "; std::string filename; std::getline(std::cin, filename); std::ifstream file(filename); if (file.is_open()) { std::stringstream buffer; buffer << file.rdbuf(); content = buffer.str(); std::cout << "Loaded " << content.length() << " characters from file." << std::endl; } else { std::cout << "Could not open file!" << std::endl; continue; } int epochs; double lr; std::cout << "Enter number of epochs: "; std::string epStr; std::getline(std::cin, epStr); epochs = std::stoi(epStr); std::cout << "Enter learning rate (e.g. 0.1): "; std::string lrStr; std::getline(std::cin, lrStr); lr = std::stod(lrStr); std::string finalData = "[SYS]" + currentSystemPrompt + content + ""; trainOnSequence(nn, tok, emb, finalData, epochs, lr); } else if (cmdIn == "/sysPrompt") { std::cout << "Current System Prompt: " << currentSystemPrompt << std::endl; std::cout << "Enter new System Prompt: "; std::getline(std::cin, currentSystemPrompt); std::cout << "System Prompt updated!" << std::endl; } else if (cmdIn == "/trainVulkan") { std::cout << nn.trainVulkan() << "\n"; } else if (cmdIn == "/help") { std::cout << "\n--- MENU ---" << std::endl; std::cout << "/train\n/trainFile\n/sysPrompt\n/help\n/exit\n"; } else if (cmdIn == "/clr") { std::cout << "\033[2J\033[1;1H"; } else { std::string prompt = "[SYS]" + currentSystemPrompt + "[USER]" + cmdIn + "[AI]"; std::vector currentTokens = tok.textToTokens(prompt); std::cout << "AI: "; long long totalParams = 0; for (int i = 0; i < numLayers - 1; i++) { long long weights = (long long)layers[i].size * layers[i + 1].size; long long biases = (long long)layers[i + 1].size; totalParams += (weights + biases); } std::string modelSizeStr; { std::stringstream ss; if (totalParams >= 1000000000000LL) ss << std::fixed << std::setprecision(1) << (double)totalParams / 1000000000000.0 << "t"; else if (totalParams >= 1000000000LL) ss << std::fixed << std::setprecision(1) << (double)totalParams / 1000000000.0 << "b"; else if (totalParams >= 1000000LL) ss << std::fixed << std::setprecision(1) << (double)totalParams / 1000000.0 << "m"; else if (totalParams >= 1000LL) ss << std::fixed << std::setprecision(1) << (double)totalParams / 1000.0 << "k"; else ss << totalParams; modelSizeStr = ss.str(); } // Переменные для замера скорости auto startTime = std::chrono::high_resolution_clock::now(); int tokensInSecond = 0; double tokensPerSec = 0; for (int g = 0; g < 1024; g++) { std::vector out = nn.feedForward(buildNetInput(currentTokens, emb)); int bestId = 0; for (int i = 0; i < MAX_VOCAB; i++) { if (out[i] > out[bestId]) bestId = i; } if (bestId == 0) break; tokensInSecond++; auto currentTime = std::chrono::high_resolution_clock::now(); std::chrono::duration elapsed = currentTime - startTime; if (elapsed.count() >= 0.1) { tokensPerSec = tokensInSecond / elapsed.count(); tokensInSecond = 0; startTime = currentTime; } std::string word = tok.getWord(bestId); std::cout << word << std::flush; std::cout << "\033[s" << "\033[999;1H" << "\033[2K" << "--- [ID: " << bestId << "] | " << "Speed: " << std::fixed << std::setprecision(1) << tokensPerSec*10 << " t/s | " << "Model: " << modelSizeStr << " params ---" << "\033[u" << std::flush; currentTokens.push_back(bestId); } // Чтобы курсор не остался внизу после генерации std::cout << std::endl; } } return 0; }