From 1c4c0f9f1f34517a51546c2585a433d888b08a01 Mon Sep 17 00:00:00 2001
From: Shuyang Li <lishuyang@meta.com>
Date: Wed, 17 May 2023 14:29:05 -0700
Subject: [PATCH] Skip transformers.models.llama* in MMF hf transformers patch
 (#1313)

Summary:
Pull Request resolved: https://github.com/facebookresearch/mmf/pull/1313

Patch prevents error `ImportError: tokenizers>=0.13.3 is required for a normal functioning of this module, but found tokenizers==0.12.1`

```
***/mmf/__init__.py in <module>
      4 from mmf.utils.patch import patch_transformers
      5
----> 6 patch_transformers()
      7
      8 from mmf import common, datasets, models, modules, utils
***/mmf/utils/patch.py in patch_transformers(log_incompatible)
     65             if not module or module == "." or module[0] == ".":
     66                 continue
---> 67             sys.modules[f"transformers.{module}"] = importlib.import_module(
     68                 f"transformers.models.{key}.{module}"
     69             )
***/runtime/lib/python3.8/importlib/__init__.py in import_module(name, package)
    137                 break
    138             level += 1
--> 139     return _bootstrap._gcd_import(name[level:], package, level)
    140
    141
***/libfb/py/import_proxy.py in wrapper(module)
     59     wraps(_exec_module)
     60     def wrapper(module: ModuleType) -> None:
---> 61         _exec_module(module)
     62         _fire_callbacks(module)
     63
***/python/parsh/autoreload/measurements.py in patched_exec_module(self_, module)
     51             start = timer()
     52             try:
---> 53                 orig_exec_module(self_, module)
     54             finally:
     55                 _IS_TOP_LEVEL_IMPORT = is_top_level_import
***/transformers/models/llama/tokenization_llama_fast.py in <module>
     22
     23
---> 24 require_version("tokenizers>=0.13.3")
     25
     26 if is_sentencepiece_available():
***/transformers/utils/versions.py in require_version(requirement, hint)
    122     if want_ver is not None:
    123         for op, want_ver in wanted.items():
--> 124             _compare_versions(op, got_ver, want_ver, requirement, pkg, hint)
    125
    126
***/transformers/utils/versions.py in _compare_versions(op, got_ver, want_ver, requirement, pkg, hint)
     48         )
     49     if not ops[op](version.parse(got_ver), version.parse(want_ver)):
---> 50         raise ImportError(
     51             f"{requirement} is required for a normal functioning of this module, but found {pkg}=={got_ver}.{hint}"
     52         )
ImportError: tokenizers>=0.13.3 is required for a normal functioning of this module, but found tokenizers==0.12.1.
```

Reviewed By: RylanC24, ebsmothers, BruceChaun

Differential Revision: D45878336

fbshipit-source-id: e4ed01806a17717837ca0f9eee13fe67d6086c7a
---
 mmf/utils/patch.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/mmf/utils/patch.py b/mmf/utils/patch.py
index f929ff3c4..2c3aa18d5 100644
--- a/mmf/utils/patch.py
+++ b/mmf/utils/patch.py
@@ -52,6 +52,10 @@ def patch_transformers(log_incompatible=False):
         if key.startswith("__"):
             continue
 
+        # Avoid LLaMa tokenizers 0.13.3 > 0.12.1 requirement as we're not using LLaMa
+        if key.startswith("llama"):
+            continue
+
         model_lib = importlib.import_module(f"transformers.models.{key}")
         if not hasattr(model_lib, "_modules"):
             if log_incompatible: