Fix deepl variable translation issue 276

rosetta/static/admin/rosetta/js/rosetta.js

@@ -24,7 +24,7 @@ $(document).ready(function () {
 
  orig = unescape(orig)
  .replace(/<br\s?\/?>/g, "\n")
- .replace(/<code>/, "")
+ .replace(/<code>/g, "")
  .replace(/<\/code>/g, "")
  .replace(/&gt;/g, ">")
  .replace(/&lt;/g, "<");

rosetta/tests/tests.py

@@ -1009,6 +1009,40 @@ def test_47_2_deeps_ajax_translation(self):
  )
  self.assertContains(r, '"Salut tout le monde"')
 
+ @vcr.use_cassette(
+ "fixtures/vcr_cassettes/test_deepl_ajax_translation_with_variables.yaml",
+ match_on=["method", "scheme", "port", "path", "query", "raw_body"],
+ record_mode="once",
+ )
+ @override_settings(
+ DEEPL_AUTH_KEY="FAKE",
+ AZURE_CLIENT_SECRET=None,
+ )
+ def test_deepl_ajax_translation_with_variables(self):
+ text = "Ci sono %(items)d %(name)s disponibili."
+ r = self.client.get(
+ reverse("rosetta.translate_text") + f"?from=it&to=en&text={text}"
+ )
+ self.assertEqual(
+ r.json().get("translation"), "There are %(items)d %(name)s available."
+ )
+
+
+ def test_formating_text_to_and_from_deepl(self):
+ from ..translate_utils import format_text_to_deepl, format_text_from_deepl
+
+ samples = [
+ "Es gibt %(items)d %(name)s verfügbar.",
+ "Ci sono %(items)d %(name)s disponibili.",
+ "Há %(items)d %(name)s disponíveis.",
+ "Stokta %(items)d %(name)s var.",
+ ]
+ for sample in samples:
+ to_deepl = format_text_to_deepl(sample)
+ from_deepl = format_text_from_deepl(to_deepl)
+ back_to_deepl = format_text_to_deepl(from_deepl)
+ self.assertEqual(to_deepl, back_to_deepl)
+
  @override_settings(ROSETTA_REQUIRES_AUTH=True)
  def test_48_requires_auth_not_respected_issue_203(self):
  self.client.logout()

rosetta/translate_utils.py

@@ -1,6 +1,6 @@
 import json
+import re
 import uuid
-
 import requests
 
 from django.conf import settings
@@ -49,8 +49,38 @@ def translate(text, from_language, to_language):
  else:
  raise TranslationException("No translation API service is configured.")
 
+def format_text_to_deepl(text):
+ pattern = r"%\((\w+)\)(\w)"
+ def replace_variable(match):
+ # Our pattern will always catch 2 groups, the first group being '%('
+ # Second group being ')d' or ')s'
+ variable = match.group(1)
+ type_specifier = match.group(2)
+ if variable and type_specifier:
+ return f'<var type="{type_specifier}">{variable}</var>'
+ else:
+ raise TranslationException("Badly formatted variable in translation")
+
+ return re.sub(pattern, replace_variable, text)
+
+def format_text_from_deepl(text):
+ for g in re.finditer(r'.*?<var type="(?P<type>[rsd])">(?P<variable>[0-9a-zA-Z_]+)</var>.*?', text):
+ t, v = g.groups()
+ text = text.replace(f'<var type="{t}">{v}</var>', f"%({v}){t}")
+ return text
+
+
 
 def translate_by_deepl(text, to_language, auth_key):
+ """
+ This method connects to the translator Deepl API and fetches a response with translations.
+ :param text: The source text to be translated
+ :param to_language: The target language to translate the text into
+ Wraps variables in <var></var> tags and instructs Deepl not to translate those.
+ Then from Deepl response, converts back these tags to django variable syntax.
+ %(name)s becomes <var type="s">name</var> and back to %(name)s in the response text.
+ :return: Returns the response from the Deepl as a python object.
+ """
  if auth_key.lower().endswith(":fx"):
  endpoint = "https://api-free.deepl.com"
  else:
@@ -60,16 +90,19 @@ def translate_by_deepl(text, to_language, auth_key):
  f"{endpoint}/v2/translate",
  headers={"Authorization": f"DeepL-Auth-Key {auth_key}"},
  data={
+ "tag_handling": "xml",
+ "ignore_tags": "var",
  "target_lang": to_language.upper(),
- "text": text,
+ "text": format_text_to_deepl(text),
  },
  )
  if r.status_code != 200:
  raise TranslationException(
  f"Deepl response is {r.status_code}. Please check your API key or try again later."
  )
+
  try:
- return r.json().get("translations")[0].get("text")
+ return format_text_from_deepl(r.json().get("translations")[0].get("text"))
  except Exception:
  raise TranslationException("Deepl returned a non-JSON or unexpected response.")
 

testproject/fixtures/vcr_cassettes/test_47_2_deeps_ajax_translation.yaml

@@ -1,6 +1,6 @@
 interactions:
 - request:
- body: target_lang=FR&text=hello+world
+ body: tag_handling=xml&ignore_tags=var&target_lang=FR&text=hello+world
  headers:
  Accept:
  - '*/*'

testproject/fixtures/vcr_cassettes/test_deepl_ajax_translation_with_variables.yaml

@@ -0,0 +1,46 @@
+interactions:
+- request:
+ body: tag_handling=xml&ignore_tags=var&target_lang=EN&text=Ci+sono+%3Cvar+type%3D%22d%22%3Eitems%3C%2Fvar%3E+%3Cvar+type%3D%22s%22%3Ename%3C%2Fvar%3E+disponibili.
+ headers:
+ Accept:
+ - '*/*'
+ Accept-Encoding:
+ - gzip, deflate
+ Authorization:
+ - DeepL-Auth-Key FAKE
+ Connection:
+ - keep-alive
+ Content-Length:
+ - '156'
+ Content-Type:
+ - application/x-www-form-urlencoded
+ User-Agent:
+ - python-requests/2.32.3
+ method: POST
+ uri: https://api-free.deepl.com/v2/translate
+ response:
+ body:
+ string: '{"translations": [{"detected_source_language": "IT", "text": "There are <var type=\"d\">items</var> <var type=\"s\">name</var> available."}]}'
+ headers:
+ access-control-allow-origin:
+ - '*'
+ access-control-expose-headers:
+ - Server-Timing, X-Trace-ID
+ content-type:
+ - application/json
+ date:
+ - Mon, 07 Oct 2024 07:54:03 GMT
+ server-timing:
+ - l7_lb_tls;dur=77, l7_lb_idle;dur=2, l7_lb_receive;dur=1, l7_lb_total;dur=124
+ strict-transport-security:
+ - max-age=63072000; includeSubDomains; preload
+ transfer-encoding:
+ - chunked
+ vary:
+ - Accept-Encoding
+ x-trace-id:
+ - 0ec04e3eef784472bedc5be15a1f8259
+ status:
+ code: 200
+ message: OK
+version: 1