Add Lapa Ukrainian HW-OCR LoRA adapter v1

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,119 @@
+---
+base_model: lapa-llm/lapa-v0.1.2-instruct
+library_name: peft
+license: gemma
+pipeline_tag: image-text-to-text
+language:
+- uk
+tags:
+- lora
+- peft
+- ocr
+- handwriting
+- handwritten-text-recognition
+- ukrainian
+- gemma3
+- vision-language
+---
+
+# Lapa Ukrainian Handwriting OCR — LoRA Adapter
+
+LoRA adapter on top of [`lapa-llm/lapa-v0.1.2-instruct`](https://huggingface.co/lapa-llm/lapa-v0.1.2-instruct)
+(a Gemma-3-12B Ukrainian vision-language model) for **Ukrainian handwritten-text
+recognition (HTR / OCR)** on document crops.
+
+The base Lapa model, applied zero-shot to handwriting crops, tends to *paraphrase*
+rather than transcribe literally. This adapter retrains the text decoder to emit a
+literal transcription of the text in the image. It was developed as an OCR component
+for a Ukrainian HTR pipeline (handwritten + printed regions, math formulas).
+
+## Results (internal validation)
+
+| Metric | Base Lapa (bf16) | + this LoRA |
+|---|---|---|
+| Handwritten CER | 3.28 | **0.113** |
+| Handwritten exact-match | 1.3% | **47.7%** |
+| Printed CER | 1.08 | **0.187** |
+
+CER > 1 on the base reflects heavy paraphrasing (output far longer than ground truth).
+The adapter removes that behavior and produces faithful transcriptions.
+
+## Intended use
+
+- Transcribing **Ukrainian handwritten / printed text crops** (region-level images,
+  not full pages) into plain text.
+- As a cross-vote / ensemble OCR partner alongside other VLMs.
+
+Not tuned for: full-page layout, non-Ukrainian scripts, or marginal / very low-quality
+regions (CER rises to ~0.55 on hard, low-confidence crops).
+
+## How to use
+
+```python
+import torch
+from PIL import Image
+from peft import PeftModel
+from transformers import AutoModelForImageTextToText, AutoProcessor
+
+BASE = "lapa-llm/lapa-v0.1.2-instruct"
+ADAPTER = "lapa-llm/lapa-ocr-lora"  # this repo
+
+base = AutoModelForImageTextToText.from_pretrained(
+    BASE,
+    torch_dtype=torch.bfloat16,
+    device_map="auto",
+    attn_implementation="sdpa",
+)
+model = PeftModel.from_pretrained(base, ADAPTER).eval()
+processor = AutoProcessor.from_pretrained(BASE)
+
+PROMPT = "Transcribe Ukrainian text literally. Output only the text, no preamble."
+img = Image.open("crop.png").convert("RGB")
+messages = [{
+    "role": "user",
+    "content": [
+        {"type": "image", "image": img},
+        {"type": "text", "text": PROMPT},
+    ],
+}]
+
+inputs = processor.apply_chat_template(
+    messages, add_generation_prompt=True, tokenize=True,
+    return_dict=True, return_tensors="pt", padding=True,
+).to(model.device, dtype=torch.bfloat16)
+
+with torch.inference_mode():
+    gen = model.generate(**inputs, max_new_tokens=256, do_sample=False, num_beams=1)
+text = processor.batch_decode(
+    gen[:, inputs["input_ids"].shape[1]:], skip_special_tokens=True
+)[0].strip()
+print(text)
+```
+
+## Training
+
+- **Base:** `lapa-llm/lapa-v0.1.2-instruct` (vision tower frozen; text decoder adapted)
+- **Method:** LoRA (PEFT) — r=64, alpha=128, dropout=0.05, bias=none
+- **Target modules:** `q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj`
+- **Task type:** `CAUSAL_LM`
+- **Epochs:** 5 · **LR:** 1e-4 · **batch:** 2 × grad-accum 4 · **max_seq_len:** 1024
+- **Precision:** bf16 · **Hardware:** 1× H100 80GB
+- **Data:** Ukrainian handwritten / printed text crops with literal transcriptions.
+
+## License
+
+This adapter is a derivative of Gemma-3 (via Lapa) and is released under the
+**[Gemma Terms of Use](https://ai.google.dev/gemma/terms)**. Use is subject to the
+[Gemma Prohibited Use Policy](https://ai.google.dev/gemma/prohibited_use_policy).
+You must comply with the base model's license; see
+[`lapa-llm/lapa-v0.1.2-instruct`](https://huggingface.co/lapa-llm/lapa-v0.1.2-instruct).
+
+## Acknowledgements
+
+Built on the [Lapa LLM](https://huggingface.co/lapa-llm) by the Ukrainian Catholic
+University, AGH University of Krakow, Igor Sikorsky Kyiv Polytechnic Institute, and
+Lviv Polytechnic. Base model: Gemma-3-12B (Google DeepMind).
+
+### Framework versions
+- PEFT 0.19.1
+- Transformers (Gemma-3 support: ≥ 4.50)

adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "lapa-llm/lapa-v0.1.2-instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "gate_proj",
+    "down_proj",
+    "up_proj",
+    "q_proj",
+    "v_proj",
+    "o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8deff408306a5226ab731092678672ea61194af4bbdd1beb5bb2d06cf58ea85
+size 1095430088

chat_template.jinja

@@ -0,0 +1,47 @@
+{{ bos_token }}
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {%- set first_user_prefix = messages[0]['content'] + '
+
+' -%}
+    {%- else -%}
+        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '
+
+' -%}
+    {%- endif -%}
+    {%- set loop_messages = messages[1:] -%}
+{%- else -%}
+    {%- set first_user_prefix = "" -%}
+    {%- set loop_messages = messages -%}
+{%- endif -%}
+{%- for message in loop_messages -%}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
+        {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
+    {%- endif -%}
+    {%- if (message['role'] == 'assistant') -%}
+        {%- set role = "model" -%}
+    {%- else -%}
+        {%- set role = message['role'] -%}
+    {%- endif -%}
+    {{ '<start_of_turn>' + role + '
+' + (first_user_prefix if loop.first else "") }}
+    {%- if message['content'] is string -%}
+        {{ message['content'] | trim }}
+    {%- elif message['content'] is iterable -%}
+        {%- for item in message['content'] -%}
+            {%- if item['type'] == 'image' -%}
+                {{ '<start_of_image>' }}
+            {%- elif item['type'] == 'text' -%}
+                {{ item['text'] | trim }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{ raise_exception("Invalid content type") }}
+    {%- endif -%}
+    {{ '<end_of_turn>
+' }}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{'<start_of_turn>model
+'}}
+{%- endif -%}

processor_config.json

@@ -0,0 +1,28 @@
+{
+  "image_processor": {
+    "do_convert_rgb": null,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_processor_type": "Gemma3ImageProcessor",
+    "image_seq_length": 256,
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "resample": 2,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "height": 896,
+      "width": 896
+    }
+  },
+  "image_seq_length": 256,
+  "processor_class": "Gemma3Processor"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96c223ece326885174a741fcb28446f49757a4f491b1187cdeb312c13b98b1ba
+size 37379975

tokenizer_config.json

@@ -0,0 +1,25 @@
+{
+  "backend": "tokenizers",
+  "boi_token": "<start_of_image>",
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eoi_token": "<end_of_image>",
+  "eos_token": "<eos>",
+  "image_token": "<image_soft_token>",
+  "is_local": false,
+  "local_files_only": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "boi_token": "<start_of_image>",
+    "eoi_token": "<end_of_image>",
+    "image_token": "<image_soft_token>"
+  },
+  "pad_token": "<pad>",
+  "processor_class": "Gemma3Processor",
+  "sp_model_kwargs": null,
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}