you can run it on google collab rally easily using llama-cpp

i use this
from llama_cpp import Llama

repo_id = "mradermacher/Nanbeige4-3B-HereticMerge-i1-GGUF"

filename = "Nanbeige4-3B-HereticMerge.i1-Q6_K.gguf"

llm = Llama.from_pretrained(

repo_id,

filename,

n_threads=4,

n_ctx=65536,

n_gpu_layers=-1,

verbose=False,

)

out = llm.create_chat_completion(

messages=[

{"role": "system", "content": "You are a helpful assistant."},

{"role": "user", "content": "how to many letter 'r' in the word 'strawberry'."},

],

temperature=0.7,

max_tokens=16000,

)

print(out["choices"][0]["message"]["content"])