From 2938fa5c77b798502b48471227f569aff5bca8f8 Mon Sep 17 00:00:00 2001 From: Allan Stockman Rugano Date: Wed, 3 Jul 2024 22:59:03 +0200 Subject: [PATCH 1/3] QR image --- .../doc_templates/program_receipt.pdf | Bin 660414 -> 661490 bytes .../apps/power_query/processors.py | 83 +++++++++--------- .../apps/power_query/utils.py | 38 +++++++- tests/power_query/test_pq_utils.py | 16 ++++ 4 files changed, 90 insertions(+), 47 deletions(-) diff --git a/src/hope_country_report/apps/power_query/doc_templates/program_receipt.pdf b/src/hope_country_report/apps/power_query/doc_templates/program_receipt.pdf index 0d24454bf0bfb3982d8199d9ec65e94ee652de17..33d11e9837d3f202a02e727d26e2c90268d62ed2 100644 GIT binary patch delta 1422 zcmah}Z)_Ar6lbpG4LO{eGZE7&s+@GzzQo7f@ zJ181bv}rU^h&93a;1?4Dd@02!@gN2@#t2_*OszH=|9nuay{a?{R@0{EELOdGCT{ZE zo&CM{X5Ra~xBK?lx}VS2y*Mc;BcIfY+rpvE8YZ&NC{y=sh2Gs}JQPVAs4nDJutM?Q z#vv{}%Ox8Qff9k+1e^k^1xf|VvRneMUy~@`|2tUEKgxk!+0kk6t|XYj1;O0@BM9-U zGhlAnB`aJuJC?0kVZBhokNpZB%+Ab#8<1aI1QmR82E_Om3!t3mZh}pvF*9y-_FFvv z9oSiLP_0vCjHvvDX;AGTIzKlJx_1yC!YGUy8lX39#M0<7r0B$hiOMh{s_r4O4`W1i z-9w2cR!mhrib}<5KNLYmU)+rLK(X7=p8)0gSJ(UxSZ+0ZL1O2B_ndgJ!Q`u7w zuk9G`4pzaVop*j!rnf%-^2nKwXO}8AeLMZl8;_sB`M2IXG_|Vq)eDLF$){c44?l3a zX)XI@NMGF9UekXn-gkA(n_C|ao=u#12^=37`Ls2bTetC1x-L<2ZR|Pq^6*&BGuz&J zY4r1{5AiGcE7I76d+u=g>Wk86l@YkGh1J|6ew#JB4ITnpZ+^Vh^ZW`B^J4!Ue%}BhBSrq!*5>&eiX~urz+vs(x7PEN}Ga-=*ho0 zqOc-P(~A5E{tu3kqCAO@&_DMZ{3j_&<)UOL!9@pCU$Gxkip5|I7m2}`D#o&)xEi4v z(+ew3TaiSsVZr)(=-r!m(m<5*#wu9d8Pa7+#E&7^tc7&8ttH50jcvnCa!6Bj%#?=u d|Mj@_eWcP6D?MNtT~JjqWenEVHt%VHe*mOVt>FLw delta 1150 zcmah|TWB0r7-q7U_WXp>l2XwqIgGj13^Qlu%*@WZ8*|yLiA{-aBSJLDY-VCw)1B?? zuC3-FidwNKnul@vkhgkH#xA?KHIq~ol`8)RUuME< zObAiK5#fk(#5m#{YdF>hOh6xB5OntKpC}dVy@W0bD@I-L;u1>SFj9<1nuGeE(f2pQ zjEDPz+%KpY5ft|B6(0WeGP(+zOsrrpzXMV!EDyMaXUgN4P=o!jgv6;2-%79hRH@DG zo)KIASZuuBfB3}dyUAk+z5ZVNQ!~-==>y~S4;*X1kbbc4$68@?}aD&2E0ojo!R{3#Hp^W(*tv-AA0oFUFSZ??DdCF z-t+k~z4nGUx398z*ZG~3l?8fc!)#}6;p)@ddWYUwcr3CKCOApw}d&p6oDOVF#`ATR@Fq|OM$RcQFxZk#$Y3VJ|2S>&5#HgB1)2~ zNj_E)@7Y;Q_V_LzY&hJs7>-s_R}d0cHXO$)1rur+}Pqi zzuWVj(oK8Fw~IyBNsC!~qU5snIJELq9*@IjVMU_$b_sSk4t+sn4M;U?5J6+3oiJV9 zprewOcPxo2E|v1Ctx1kEO7fOzyNY8mhCpUAf`;G=1Z}|eFV{g&r%p^2E8)Qf4B_3- zb=6d=eUvvgJh$<*gdUVO!Z~58N`tqE4{tSJ)zm6o)ikPjn(`k$T{Z61(-P{9oThFi zMKzw{9UrDTsY;&erdGwXH2s#Z?AzNXT%Z3vAaC3Lf{SUmOP "ProcessorResult": class ToFormPDF(ProcessorStrategy): + """ + Produce reports in PDF form or cards. + This class handles PDF generation with special attention to QR code fields. + """ + file_suffix = ".pdf" format = TYPE_DETAIL needs_file = True @@ -197,37 +203,32 @@ class ToFormPDF(ProcessorStrategy): def process(self, context: Dict[str, Any]) -> bytes: tpl = self.formatter.template reader = PdfReader(tpl.doc) - - font_size = context.get("context", {}).get("font_size", 10) - font_color = context.get("context", {}).get("font_color", "black") ds = to_dataset(context["dataset"].data).dict output_pdf = PdfWriter() + for index, entry in enumerate(ds, start=1): with NamedTemporaryFile(suffix=".pdf", delete=True) as temp_pdf_file: writer = PdfWriter() text_values = {} - special_values = {} images = {} - try: - for page in reader.pages: - for annot in page.annotations: - annot = annot.get_object() - field_name = annot[FieldDictionaryAttributes.T] - if field_name in entry: - value = entry[field_name] - language = self.is_special_language_field(field_name) - if self.is_image_field(annot): - rect = annot[AnnotationDictionaryAttributes.Rect] - text_values[field_name] = None - images[field_name] = [rect, value] - elif language: - special_values[field_name] = {"value": value, "language": language} - else: - text_values[field_name] = value - except IndexError as exc: - capture_exception(exc) - logger.exception(exc) - raise + qr_codes = {} + + # Handle annotations for text, images, and QR codes + for page in reader.pages: + for annot in page.annotations: + annot = annot.get_object() + field_name = annot[FieldDictionaryAttributes.T] + if field_name in entry: + value = entry[field_name] + if field_name.endswith("_qr"): + qr_codes[field_name] = value # Handle QR codes differently + elif self.is_image_field(annot): + rect = annot[AnnotationDictionaryAttributes.Rect] + images[field_name] = (rect, value) + else: + text_values[field_name] = value + + # Update PDF with text values writer.append(reader) writer.update_page_form_field_values(writer.pages[-1], text_values, flags=FieldFlag.READ_ONLY) output_stream = io.BytesIO() @@ -235,26 +236,26 @@ def process(self, context: Dict[str, Any]) -> bytes: output_stream.seek(0) temp_pdf_file.write(output_stream.read()) + # Open processed document for image and QR code insertion document = fitz.open(stream=output_stream.getvalue(), filetype="pdf") - for field_name, text in special_values.items(): - insert_special_image(document, field_name, text, font_size, font_color) - for field_name, (rect, image_path) in images.items(): - if image_path: - self.insert_external_image(document, field_name, image_path) - else: - logger.warning(f"Image not found for field: {field_name}") - document.ez_save(temp_pdf_file.name, deflate_fonts=True, deflate_images=1, deflate=1) + self.insert_images_and_qr_codes(document, images, qr_codes) + document.save(temp_pdf_file.name) output_stream.seek(0) output_pdf.append_pages_from_reader(PdfReader(temp_pdf_file.name)) + output_stream = io.BytesIO() output_pdf.write(output_stream) output_stream.seek(0) - fitz_pdf_document = fitz.open(stream=output_stream, filetype="pdf") - # Convert the PDF to an image-based PDF - image_pdf_bytes = convert_pdf_to_image_pdf(fitz_pdf_document, dpi=300) + fitz_pdf_document = fitz.open("pdf", output_stream.read()) + return convert_pdf_to_image_pdf(fitz_pdf_document, dpi=300) - return image_pdf_bytes + def insert_images_and_qr_codes(self, document, images, qr_codes): + for field_name, (rect, image_path) in images.items(): + self.insert_external_image(document, field_name, image_path, rect) + for field_name, data in qr_codes.items(): + rect, page_index = get_field_rect(document, field_name) + insert_qr_code(document, field_name, data, rect, page_index) def insert_external_image(self, document: fitz.Document, field_name: str, image_path: str, font_size: int = 10): """ @@ -296,11 +297,7 @@ def is_image_field(self, annot: ArrayObject) -> bool: """ Checks if a given PDF annotation represents an image field. """ - return ( - annot.get(FieldDictionaryAttributes.FT) == "/Btn" - and AnnotationDictionaryAttributes.P in annot - and AnnotationDictionaryAttributes.AP in annot - ) + return annot.get(FieldDictionaryAttributes.FT) == "/Btn" and AnnotationDictionaryAttributes.AP in annot def is_special_language_field(self, field_name: str) -> Optional[str]: """Extract language code from the field name if it exists.""" @@ -311,7 +308,7 @@ def is_special_language_field(self, field_name: str) -> Optional[str]: return None def load_image_from_blob_storage(self, image_path: str) -> BytesIO: - with HopeStorage().open(image_path, "rb") as img_file: + with DataSetStorage().open(image_path, "rb") as img_file: return BytesIO(img_file.read()) diff --git a/src/hope_country_report/apps/power_query/utils.py b/src/hope_country_report/apps/power_query/utils.py index 5a800063..1b87b6d0 100644 --- a/src/hope_country_report/apps/power_query/utils.py +++ b/src/hope_country_report/apps/power_query/utils.py @@ -1,5 +1,6 @@ from typing import Any, Dict, Optional, TYPE_CHECKING, Union from pathlib import Path +import qrcode from io import BytesIO from django.conf import settings import base64 @@ -15,7 +16,7 @@ from django.db.models import QuerySet from django.http import HttpRequest, HttpResponse from django.utils.safestring import mark_safe - +import io import fitz import tablib from constance import config @@ -214,10 +215,11 @@ def convert_pdf_to_image_pdf(pdf_document: fitz.Document, dpi: int = 300) -> byt """ new_pdf_document = fitz.open() - for page_num in range(len(pdf_document)): + for page_num in range(pdf_document.page_count): # Use .page_count here pix = pdf_document[page_num].get_pixmap(dpi=dpi) - new_pdf_document.new_page(width=pix.width, height=pix.height) - new_pdf_document[page_num].insert_image(fitz.Rect(0, 0, pix.width, pix.height), pixmap=pix) + new_page = new_pdf_document.new_page(width=pix.width, height=pix.height) + new_page.insert_image(fitz.Rect(0, 0, pix.width, pix.height), pixmap=pix) + new_pdf_bytes = BytesIO() new_pdf_document.save(new_pdf_bytes, deflate_fonts=1, deflate_images=1, deflate=1) new_pdf_bytes.seek(0) @@ -241,3 +243,31 @@ def insert_special_image( page.insert_image(img_rect, stream=image_stream, keep_proportion=False) else: logger.info(f"Field {field_name} not found") + + +def insert_qr_code(document: fitz.Document, field_name: str, data: str, rect: fitz.Rect, page_index: int): + """ + Generates a QR code and inserts it into the specified field. + """ + qr = qrcode.QRCode( + version=1, + error_correction=qrcode.constants.ERROR_CORRECT_L, + box_size=10, + border=2, + ) + qr.add_data(data) + qr.make(fit=True) + + qr_image = qr.make_image(fill_color="black", back_color="white") + image_stream = io.BytesIO() + qr_image.save(image_stream, format="PNG") + image_stream.seek(0) + + page = document[page_index] + + for widget in page.widgets(): + if widget.field_name == field_name: + page.delete_widget(widget) + break + + page.insert_image(rect, stream=image_stream, keep_proportion=False) diff --git a/tests/power_query/test_pq_utils.py b/tests/power_query/test_pq_utils.py index e339dcbf..c9dc25be 100644 --- a/tests/power_query/test_pq_utils.py +++ b/tests/power_query/test_pq_utils.py @@ -25,6 +25,7 @@ insert_special_language_image, convert_pdf_to_image_pdf, get_field_rect, + insert_qr_code, ) TEST_PDF = resource_path("apps/power_query/doc_templates/program_receipt.pdf") @@ -222,3 +223,18 @@ def test_insert_special_image(sample_pdf: fitz.Document) -> None: annotations = [annot for annot in page.annots()] assert len(annotations) == 0, "No annotations found after insertion" + + +def test_insert_qr_code(sample_pdf): + field_name = "code_qr" + data = "https://example.com" + page = sample_pdf[0] + rect, page_index = get_field_rect(sample_pdf, field_name) + + if rect is not None and page_index is not None: + insert_qr_code(sample_pdf, field_name, data, rect, page_index) + page = sample_pdf[page_index] + images = page.get_images(full=True) + assert len(images) > 0, "No images found on the page, QR code insertion failed" + else: + pytest.fail("No valid rectangle or page index found for the field.") From c3714a25b0111b44cb34ca05513676946365e0e8 Mon Sep 17 00:00:00 2001 From: Allan Stockman Rugano Date: Wed, 3 Jul 2024 23:03:06 +0200 Subject: [PATCH 2/3] Storage fix --- src/hope_country_report/apps/power_query/processors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hope_country_report/apps/power_query/processors.py b/src/hope_country_report/apps/power_query/processors.py index e54d806b..0fc33564 100644 --- a/src/hope_country_report/apps/power_query/processors.py +++ b/src/hope_country_report/apps/power_query/processors.py @@ -308,7 +308,7 @@ def is_special_language_field(self, field_name: str) -> Optional[str]: return None def load_image_from_blob_storage(self, image_path: str) -> BytesIO: - with DataSetStorage().open(image_path, "rb") as img_file: + with HopeStorage().open(image_path, "rb") as img_file: return BytesIO(img_file.read()) From c25c47a8cbb2598d39da11acd7db0886f8d9532e Mon Sep 17 00:00:00 2001 From: Allan Stockman Rugano Date: Fri, 5 Jul 2024 12:07:13 +0200 Subject: [PATCH 3/3] Put back font customization --- .../apps/power_query/processors.py | 38 ++++++++++++------- .../apps/power_query/utils.py | 17 +++++---- tests/extras/testutils/factories/__init__.py | 1 - .../extras/testutils/factories/power_query.py | 2 + tests/power_query/test_pq_utils.py | 20 +++++----- 5 files changed, 47 insertions(+), 31 deletions(-) diff --git a/src/hope_country_report/apps/power_query/processors.py b/src/hope_country_report/apps/power_query/processors.py index 0fc33564..4071ef1a 100644 --- a/src/hope_country_report/apps/power_query/processors.py +++ b/src/hope_country_report/apps/power_query/processors.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, TYPE_CHECKING +from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING import io import logging @@ -6,7 +6,6 @@ import re from collections.abc import Callable from io import BytesIO -from pathlib import Path from django.core.files.temp import NamedTemporaryFile from django.template import Context, Template @@ -22,20 +21,17 @@ from strategy_field.registry import Registry from strategy_field.utils import fqn -from hope_country_report.apps.power_query.storage import HopeStorage, DataSetStorage - +from hope_country_report.apps.power_query.storage import HopeStorage from hope_country_report.apps.power_query.utils import ( - get_field_rect, - to_dataset, convert_pdf_to_image_pdf, - insert_special_image, + get_field_rect, insert_qr_code, + insert_special_image, + to_dataset, ) logger = logging.getLogger(__name__) if TYPE_CHECKING: - from typing import Tuple - from .models import Dataset, Formatter, ReportTemplate ProcessorResult = bytes | BytesIO @@ -203,6 +199,8 @@ class ToFormPDF(ProcessorStrategy): def process(self, context: Dict[str, Any]) -> bytes: tpl = self.formatter.template reader = PdfReader(tpl.doc) + font_size = context.get("context", {}).get("font_size", 10) + font_color = context.get("context", {}).get("font_color", "black") ds = to_dataset(context["dataset"].data).dict output_pdf = PdfWriter() @@ -210,25 +208,27 @@ def process(self, context: Dict[str, Any]) -> bytes: with NamedTemporaryFile(suffix=".pdf", delete=True) as temp_pdf_file: writer = PdfWriter() text_values = {} + special_values = {} images = {} qr_codes = {} - # Handle annotations for text, images, and QR codes for page in reader.pages: for annot in page.annotations: annot = annot.get_object() field_name = annot[FieldDictionaryAttributes.T] if field_name in entry: value = entry[field_name] + language = self.is_special_language_field(field_name) if field_name.endswith("_qr"): - qr_codes[field_name] = value # Handle QR codes differently + qr_codes[field_name] = value elif self.is_image_field(annot): rect = annot[AnnotationDictionaryAttributes.Rect] images[field_name] = (rect, value) + elif language: + special_values[field_name] = {"value": value, "language": language} else: text_values[field_name] = value - # Update PDF with text values writer.append(reader) writer.update_page_form_field_values(writer.pages[-1], text_values, flags=FieldFlag.READ_ONLY) output_stream = io.BytesIO() @@ -238,7 +238,7 @@ def process(self, context: Dict[str, Any]) -> bytes: # Open processed document for image and QR code insertion document = fitz.open(stream=output_stream.getvalue(), filetype="pdf") - self.insert_images_and_qr_codes(document, images, qr_codes) + self.insert_images_and_qr_codes(document, images, qr_codes, special_values, font_size, font_color) document.save(temp_pdf_file.name) output_stream.seek(0) output_pdf.append_pages_from_reader(PdfReader(temp_pdf_file.name)) @@ -250,7 +250,17 @@ def process(self, context: Dict[str, Any]) -> bytes: fitz_pdf_document = fitz.open("pdf", output_stream.read()) return convert_pdf_to_image_pdf(fitz_pdf_document, dpi=300) - def insert_images_and_qr_codes(self, document, images, qr_codes): + def insert_images_and_qr_codes( + self, + document: fitz.Document, + images: Dict[str, Tuple[fitz.Rect, str]], + qr_codes: Dict[str, str], + special_values: Dict[str, Dict[str, str]], + font_size: int, + font_color: str, + ): + for field_name, text in special_values.items(): + insert_special_image(document, field_name, text, int(font_size), font_color) for field_name, (rect, image_path) in images.items(): self.insert_external_image(document, field_name, image_path, rect) for field_name, data in qr_codes.items(): diff --git a/src/hope_country_report/apps/power_query/utils.py b/src/hope_country_report/apps/power_query/utils.py index 1b87b6d0..ce9bebea 100644 --- a/src/hope_country_report/apps/power_query/utils.py +++ b/src/hope_country_report/apps/power_query/utils.py @@ -1,25 +1,28 @@ -from typing import Any, Dict, Optional, TYPE_CHECKING, Union -from pathlib import Path -import qrcode -from io import BytesIO -from django.conf import settings +from typing import Any, Dict, Optional, TYPE_CHECKING + import base64 import binascii import datetime import hashlib +import io import json import logging from collections.abc import Callable, Iterable from functools import wraps -from PIL import Image, ImageDraw, ImageFont +from io import BytesIO +from pathlib import Path + +from django.conf import settings from django.contrib.auth import authenticate from django.db.models import QuerySet from django.http import HttpRequest, HttpResponse from django.utils.safestring import mark_safe -import io + import fitz +import qrcode import tablib from constance import config +from PIL import Image, ImageDraw, ImageFont from sentry_sdk import configure_scope if TYPE_CHECKING: diff --git a/tests/extras/testutils/factories/__init__.py b/tests/extras/testutils/factories/__init__.py index 2546c32f..cacd7e1f 100644 --- a/tests/extras/testutils/factories/__init__.py +++ b/tests/extras/testutils/factories/__init__.py @@ -3,7 +3,6 @@ from .base import AutoRegisterModelFactory, factories_registry, HopeAutoRegisterModelFactory, TAutoRegisterModelFactory # isort: split -from .adv_filters import AdvancedFilterFactory from .contenttypes import * from .django_auth import * from .django_celery_beat import * diff --git a/tests/extras/testutils/factories/power_query.py b/tests/extras/testutils/factories/power_query.py index 94dfe0f9..23220e59 100644 --- a/tests/extras/testutils/factories/power_query.py +++ b/tests/extras/testutils/factories/power_query.py @@ -3,8 +3,10 @@ from django.apps import apps from django.core.files.base import ContentFile from django.core.files.uploadedfile import SimpleUploadedFile + import factory from strategy_field.utils import fqn + from hope_country_report.apps.power_query.models import ( ChartPage, Dataset, diff --git a/tests/power_query/test_pq_utils.py b/tests/power_query/test_pq_utils.py index c9dc25be..e1149189 100644 --- a/tests/power_query/test_pq_utils.py +++ b/tests/power_query/test_pq_utils.py @@ -1,32 +1,34 @@ import datetime from base64 import b64encode +from io import BytesIO from pathlib import Path -from typing import Optional, Generator, Tuple + import pytest from unittest.mock import MagicMock -from hope_country_report.utils.media import resource_path + from django.contrib.auth.models import AnonymousUser from django.http import HttpResponse from django.utils import timezone + import fitz -from PIL import Image -from io import BytesIO import tablib +from PIL import Image from pytz import utc from hope_country_report.apps.power_query.utils import ( basicauth, + convert_pdf_to_image_pdf, + get_field_rect, get_sentry_url, + insert_qr_code, + insert_special_image, + insert_special_language_image, is_valid_template, sentry_tags, sizeof, to_dataset, - insert_special_image, - insert_special_language_image, - convert_pdf_to_image_pdf, - get_field_rect, - insert_qr_code, ) +from hope_country_report.utils.media import resource_path TEST_PDF = resource_path("apps/power_query/doc_templates/program_receipt.pdf")