From 1dd8bfabe3d16cc5fb0a3e0104dc8fec9b8e632a Mon Sep 17 00:00:00 2001 From: ihor-palii Date: Tue, 25 Jul 2023 21:36:14 +0300 Subject: [PATCH 1/2] Added changes to hide sensitive information for twilio channels --- temba/channels/models.py | 43 ++++++++++++------- temba/channels/types/twilio/type.py | 3 ++ .../types/twilio_messaging_service/type.py | 3 ++ temba/channels/types/twilio_whatsapp/type.py | 3 ++ temba/channels/types/twiml_api/type.py | 3 ++ temba/utils/redact.py | 26 +++++++++++ 6 files changed, 65 insertions(+), 16 deletions(-) diff --git a/temba/channels/models.py b/temba/channels/models.py index f96738353c5..25ae1e9e89f 100644 --- a/temba/channels/models.py +++ b/temba/channels/models.py @@ -93,6 +93,8 @@ class IVRProtocol(Enum): redact_request_keys = set() redact_response_keys = set() + force_redact_request_keys = set() + force_redact_response_keys = set() def is_available_to(self, user): """ @@ -1243,39 +1245,48 @@ def get_request_display(self, user, anon_mask): Gets the request trace as it should be displayed to the given user """ redact_keys = Channel.get_type_from_code(self.channel.channel_type).redact_request_keys + force_redact_keys = Channel.get_type_from_code(self.channel.channel_type).force_redact_request_keys - return self._get_display_value(user, self.request, anon_mask, redact_keys) + return self._get_display_value(user, self.request, anon_mask, redact_keys, force_redact_keys) def get_response_display(self, user, anon_mask): """ Gets the response trace as it should be displayed to the given user """ redact_keys = Channel.get_type_from_code(self.channel.channel_type).redact_response_keys + force_redact_keys = Channel.get_type_from_code(self.channel.channel_type).force_redact_response_keys - return self._get_display_value(user, self.response, anon_mask, redact_keys) + return self._get_display_value(user, self.response, anon_mask, redact_keys, force_redact_keys) - def _get_display_value(self, user, original, mask, redact_keys=()): + def _get_display_value(self, user, original, mask, redact_keys=(), force_redact_keys=()): """ Get a part of the log which may or may not have to be redacted to hide sensitive information in anon orgs """ - if not self.channel.org.is_anon or user.has_org_perm(self.channel.org, "contacts.contact_break_anon"): - return original + def basic_reduction(): + if not self.channel.org.is_anon or user.has_org_perm(self.channel.org, "contacts.contact_break_anon"): + return original - # if this log doesn't have a msg then we don't know what to redact, so redact completely - if not self.msg_id: - return mask + # if this log doesn't have a msg then we don't know what to redact, so redact completely + if not self.msg_id: + return mask - needle = self.msg.contact_urn.path + needle = self.msg.contact_urn.path - if redact_keys: - redacted = redact.http_trace(original, needle, mask, redact_keys) - else: - redacted = redact.text(original, needle, mask) + if redact_keys: + result = redact.http_trace(original, needle, mask, redact_keys) + else: + result = redact.text(original, needle, mask) + + # if nothing was redacted, don't risk returning sensitive information we didn't find + if original == result: + return mask + + return result - # if nothing was redacted, don't risk returning sensitive information we didn't find - if original == redacted: - return mask + redacted = basic_reduction() + if force_redact_keys and self.msg_id: + redacted = redact.http_trace(redacted, self.msg.contact_urn.path, mask, force_redact_keys) return redacted diff --git a/temba/channels/types/twilio/type.py b/temba/channels/types/twilio/type.py index 012a6c6c3c9..0c43e96d32c 100644 --- a/temba/channels/types/twilio/type.py +++ b/temba/channels/types/twilio/type.py @@ -45,6 +45,9 @@ class TwilioType(ChannelType): "CalledZip", } + force_redact_request_keys = {"MessagingServiceSid"} + force_redact_response_keys = {"MessagingServiceSid"} + def is_recommended_to(self, user): org = user.get_org() countrycode = timezone_to_country_code(org.timezone) diff --git a/temba/channels/types/twilio_messaging_service/type.py b/temba/channels/types/twilio_messaging_service/type.py index 41f6ee32919..bf8854ec8ca 100644 --- a/temba/channels/types/twilio_messaging_service/type.py +++ b/temba/channels/types/twilio_messaging_service/type.py @@ -47,6 +47,9 @@ class TwilioMessagingServiceType(ChannelType): attachment_support = True + force_redact_request_keys = {"MessagingServiceSid"} + force_redact_response_keys = {"MessagingServiceSid"} + def is_recommended_to(self, user): org = user.get_org() countrycode = timezone_to_country_code(org.timezone) diff --git a/temba/channels/types/twilio_whatsapp/type.py b/temba/channels/types/twilio_whatsapp/type.py index 3cad74fe789..5340b5c1b64 100644 --- a/temba/channels/types/twilio_whatsapp/type.py +++ b/temba/channels/types/twilio_whatsapp/type.py @@ -55,3 +55,6 @@ class TwilioWhatsappType(ChannelType): "CalledState", "CalledZip", } + + force_redact_request_keys = {"MessagingServiceSid"} + force_redact_response_keys = {"MessagingServiceSid"} diff --git a/temba/channels/types/twiml_api/type.py b/temba/channels/types/twiml_api/type.py index 181dd0e068e..9b38cceb663 100644 --- a/temba/channels/types/twiml_api/type.py +++ b/temba/channels/types/twiml_api/type.py @@ -62,3 +62,6 @@ class TwimlAPIType(ChannelType): description=_("Incoming messages for this channel will be sent to this endpoint."), ), ) + + force_redact_request_keys = {"MessagingServiceSid"} + force_redact_response_keys = {"MessagingServiceSid"} diff --git a/temba/utils/redact.py b/temba/utils/redact.py index 6cc6e2ecb74..2e4484a8815 100644 --- a/temba/utils/redact.py +++ b/temba/utils/redact.py @@ -1,6 +1,7 @@ import json import urllib.parse import xml.sax.saxutils +from collections import OrderedDict from urllib.parse import parse_qs, urlencode HTTP_BODY_BOUNDARY = "\r\n\r\n" @@ -41,12 +42,37 @@ def text(s, needle, mask): return s +def replace_headers(raw_data, headers_to_replace): + try: + headers = OrderedDict() + raw_headers, raw_body = raw_data.split(HTTP_BODY_BOUNDARY) + + # parse headers + for line in str(raw_headers).split("\r\n")[1:]: + key, value = line.split(": ") + headers[key] = value + + # replace needed + for key, value in headers_to_replace.items(): + if key in headers: + headers[key] = value + + # rebuild http message + raw_headers = "\r\n".join([": ".join((key, value)) for key, value in headers.items()]) + raw_message = HTTP_BODY_BOUNDARY.join((raw_headers, raw_body)) + return raw_message + except (KeyError, ValueError): + return raw_data + + def http_trace(trace, needle, mask, body_keys=()): """ Redacts a value from the given HTTP trace by replacing it with a mask. If body_keys is specified then we also try to parse the body and replace those keyed values with the mask. Bodies are parsed as JSON and URL encoded. """ + trace = replace_headers(trace, {"Authorization": "*" * 8}) + *rest, body = trace.split(HTTP_BODY_BOUNDARY) if body and body_keys: From acee342489a447050ae05656a8786a967ab2a366 Mon Sep 17 00:00:00 2001 From: ihor-palii Date: Tue, 25 Jul 2023 23:14:57 +0300 Subject: [PATCH 2/2] Fix failed tests --- temba/channels/models.py | 4 ++-- temba/utils/redact.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/temba/channels/models.py b/temba/channels/models.py index 25ae1e9e89f..f82cac0814d 100644 --- a/temba/channels/models.py +++ b/temba/channels/models.py @@ -1285,8 +1285,8 @@ def basic_reduction(): return result redacted = basic_reduction() - if force_redact_keys and self.msg_id: - redacted = redact.http_trace(redacted, self.msg.contact_urn.path, mask, force_redact_keys) + if force_redact_keys: + redacted = redact.http_trace(redacted, "********", mask, force_redact_keys) return redacted diff --git a/temba/utils/redact.py b/temba/utils/redact.py index 2e4484a8815..d93ea647dc8 100644 --- a/temba/utils/redact.py +++ b/temba/utils/redact.py @@ -46,9 +46,10 @@ def replace_headers(raw_data, headers_to_replace): try: headers = OrderedDict() raw_headers, raw_body = raw_data.split(HTTP_BODY_BOUNDARY) + split_headers = str(raw_headers).split("\r\n") # parse headers - for line in str(raw_headers).split("\r\n")[1:]: + for line in split_headers[1:]: key, value = line.split(": ") headers[key] = value @@ -58,7 +59,8 @@ def replace_headers(raw_data, headers_to_replace): headers[key] = value # rebuild http message - raw_headers = "\r\n".join([": ".join((key, value)) for key, value in headers.items()]) + first_header = split_headers[0].strip() + raw_headers = "\r\n".join([first_header] + [": ".join((key, value)) for key, value in headers.items()]) raw_message = HTTP_BODY_BOUNDARY.join((raw_headers, raw_body)) return raw_message except (KeyError, ValueError):