#ifndef XENITH_TOKENIZER_H
#define XENITH_TOKENIZER_H
#include <cstdint>
#include <string>
#include <iostream>
#include <unordered_map>
#include "./../converter.hpp"
#include "models/test_model.h"

void proccesing_promt(const std::string& text, Model& mdl, uint32_t layer_idx = 0);
Model create_test_model();

#endif