Squashed commit of the following:

commit 803cc09837acc734ccdafc7e4b1d4adc553a7355 Author: Marco Bonetti <[email protected]> Date: Mon Oct 7 11:59:30 2024 +0200 Changelog commit 4808f4abc5c6830b609c9e1bafc4ba5daf67a19c Author: Marco Bonetti <[email protected]> Date: Mon Oct 7 11:56:55 2024 +0200 fix flake8 commit 38781c1 Author: Halit Celik <[email protected]> Date: Mon Oct 7 11:53:52 2024 +0200 remove venv commit 46d4fdf Merge: b32840d 1a341ad Author: Halit Celik <[email protected]> Date: Mon Oct 7 10:53:43 2024 +0200 merge remote and fix conflict commit b32840d Author: Halit Celik <[email protected]> Date: Mon Oct 7 10:41:56 2024 +0200 rename function and fix tests commit 2dbc232 Author: Halit Celik <[email protected]> Date: Mon Sep 30 13:26:05 2024 +0200 send variable type as html attribute commit 7e1c1cf Author: Halit Celik <[email protected]> Date: Mon Jun 10 15:07:57 2024 +0200 remove content encodig gzip from vcr cassettes commit 872f081 Merge: 3cc0680 62422e4 Author: Halit Celik <[email protected]> Date: Mon Jun 10 14:46:21 2024 +0200 sync fork commit 3cc0680 Author: Halit Celik <[email protected]> Date: Mon Jun 10 14:39:37 2024 +0200 Wraps and unwraps django variables in xml tags for deepl not to translate them
mbi · Oct 7, 2024 · 6139629 · 6139629
1 parent 1a341ad
commit 6139629
Show file tree

Hide file tree

Showing 6 changed files with 123 additions and 4 deletions.
diff --git a/CHANGES b/CHANGES
@@ -5,6 +5,8 @@ Version 0.10.2 (unreleased)
 ---------------------------
 * Tests: update flake8 in tox tests
 * Format all rendered assets (html, css, js) in a pre-commit task. (PR #294, thanks @balazs-endresz)
+* Fix Deepl translations containing variables (#276, PR #290, thanks @halitcelik)
+
 
 
 Version 0.10.1

diff --git a/rosetta/static/admin/rosetta/js/rosetta.js b/rosetta/static/admin/rosetta/js/rosetta.js
@@ -24,7 +24,7 @@ $(document).ready(function () {
 
  orig = unescape(orig)
  .replace(/<br\s?\/?>/g, "\n")
- .replace(/<code>/, "")
+ .replace(/<code>/g, "")
  .replace(/<\/code>/g, "")
  .replace(/&gt;/g, ">")
  .replace(/&lt;/g, "<");

diff --git a/rosetta/tests/tests.py b/rosetta/tests/tests.py
@@ -1009,6 +1009,39 @@ def test_47_2_deeps_ajax_translation(self):
  )
  self.assertContains(r, '"Salut tout le monde"')
 
+ @vcr.use_cassette(
+ "fixtures/vcr_cassettes/test_deepl_ajax_translation_with_variables.yaml",
+ match_on=["method", "scheme", "port", "path", "query", "raw_body"],
+ record_mode="once",
+ )
+ @override_settings(
+ DEEPL_AUTH_KEY="FAKE",
+ AZURE_CLIENT_SECRET=None,
+ )
+ def test_deepl_ajax_translation_with_variables(self):
+ text = "Ci sono %(items)d %(name)s disponibili."
+ r = self.client.get(
+ reverse("rosetta.translate_text") + f"?from=it&to=en&text={text}"
+ )
+ self.assertEqual(
+ r.json().get("translation"), "There are %(items)d %(name)s available."
+ )
+
+ def test_formating_text_to_and_from_deepl(self):
+ from ..translate_utils import format_text_from_deepl, format_text_to_deepl
+
+ samples = [
+ "Es gibt %(items)d %(name)s verfügbar.",
+ "Ci sono %(items)d %(name)s disponibili.",
+ "Há %(items)d %(name)s disponíveis.",
+ "Stokta %(items)d %(name)s var.",
+ ]
+ for sample in samples:
+ to_deepl = format_text_to_deepl(sample)
+ from_deepl = format_text_from_deepl(to_deepl)
+ back_to_deepl = format_text_to_deepl(from_deepl)
+ self.assertEqual(to_deepl, back_to_deepl)
+
  @override_settings(ROSETTA_REQUIRES_AUTH=True)
  def test_48_requires_auth_not_respected_issue_203(self):
  self.client.logout()

diff --git a/rosetta/translate_utils.py b/rosetta/translate_utils.py
@@ -1,4 +1,5 @@
 import json
+import re
 import uuid
 
 import requests
@@ -50,7 +51,41 @@ def translate(text, from_language, to_language):
  raise TranslationException("No translation API service is configured.")
 
 
+def format_text_to_deepl(text):
+ pattern = r"%\((\w+)\)(\w)"
+
+ def replace_variable(match):
+ # Our pattern will always catch 2 groups, the first group being '%('
+ # Second group being ')d' or ')s'
+ variable = match.group(1)
+ type_specifier = match.group(2)
+ if variable and type_specifier:
+ return f'<var type="{type_specifier}">{variable}</var>'
+ else:
+ raise TranslationException("Badly formatted variable in translation")
+
+ return re.sub(pattern, replace_variable, text)
+
+
+def format_text_from_deepl(text):
+ for g in re.finditer(
+ r'.*?<var type="(?P<type>[rsd])">(?P<variable>[0-9a-zA-Z_]+)</var>.*?', text
+ ):
+ t, v = g.groups()
+ text = text.replace(f'<var type="{t}">{v}</var>', f"%({v}){t}")
+ return text
+
+
 def translate_by_deepl(text, to_language, auth_key):
+ """
+ This method connects to the translator Deepl API and fetches a response with translations.
+ :param text: The source text to be translated
+ :param to_language: The target language to translate the text into
+ Wraps variables in <var></var> tags and instructs Deepl not to translate those.
+ Then from Deepl response, converts back these tags to django variable syntax.
+ %(name)s becomes <var type="s">name</var> and back to %(name)s in the response text.
+ :return: Returns the response from the Deepl as a python object.
+ """
  if auth_key.lower().endswith(":fx"):
  endpoint = "https://api-free.deepl.com"
  else:
@@ -60,16 +95,19 @@ def translate_by_deepl(text, to_language, auth_key):
  f"{endpoint}/v2/translate",
  headers={"Authorization": f"DeepL-Auth-Key {auth_key}"},
  data={
+ "tag_handling": "xml",
+ "ignore_tags": "var",
  "target_lang": to_language.upper(),
- "text": text,
+ "text": format_text_to_deepl(text),
  },
  )
  if r.status_code != 200:
  raise TranslationException(
  f"Deepl response is {r.status_code}. Please check your API key or try again later."
  )
+
  try:
- return r.json().get("translations")[0].get("text")
+ return format_text_from_deepl(r.json().get("translations")[0].get("text"))
  except Exception:
  raise TranslationException("Deepl returned a non-JSON or unexpected response.")
 

diff --git a/testproject/fixtures/vcr_cassettes/test_47_2_deeps_ajax_translation.yaml b/testproject/fixtures/vcr_cassettes/test_47_2_deeps_ajax_translation.yaml
@@ -1,6 +1,6 @@
 interactions:
 - request:
- body: target_lang=FR&text=hello+world
+ body: tag_handling=xml&ignore_tags=var&target_lang=FR&text=hello+world
  headers:
  Accept:
  - '*/*'

diff --git a/testproject/fixtures/vcr_cassettes/test_deepl_ajax_translation_with_variables.yaml b/testproject/fixtures/vcr_cassettes/test_deepl_ajax_translation_with_variables.yaml
@@ -0,0 +1,46 @@
+interactions:
+- request:
+ body: tag_handling=xml&ignore_tags=var&target_lang=EN&text=Ci+sono+%3Cvar+type%3D%22d%22%3Eitems%3C%2Fvar%3E+%3Cvar+type%3D%22s%22%3Ename%3C%2Fvar%3E+disponibili.
+ headers:
+ Accept:
+ - '*/*'
+ Accept-Encoding:
+ - gzip, deflate
+ Authorization:
+ - DeepL-Auth-Key FAKE
+ Connection:
+ - keep-alive
+ Content-Length:
+ - '156'
+ Content-Type:
+ - application/x-www-form-urlencoded
+ User-Agent:
+ - python-requests/2.32.3
+ method: POST
+ uri: https://api-free.deepl.com/v2/translate
+ response:
+ body:
+ string: '{"translations": [{"detected_source_language": "IT", "text": "There are <var type=\"d\">items</var> <var type=\"s\">name</var> available."}]}'
+ headers:
+ access-control-allow-origin:
+ - '*'
+ access-control-expose-headers:
+ - Server-Timing, X-Trace-ID
+ content-type:
+ - application/json
+ date:
+ - Mon, 07 Oct 2024 07:54:03 GMT
+ server-timing:
+ - l7_lb_tls;dur=77, l7_lb_idle;dur=2, l7_lb_receive;dur=1, l7_lb_total;dur=124
+ strict-transport-security:
+ - max-age=63072000; includeSubDomains; preload
+ transfer-encoding:
+ - chunked
+ vary:
+ - Accept-Encoding
+ x-trace-id:
+ - 0ec04e3eef784472bedc5be15a1f8259
+ status:
+ code: 200
+ message: OK
+version: 1