fix:standardize_lang

ovos_core/intent_services/__init__.py

@@ -29,6 +29,7 @@
 from ovos_core.intent_services.stop_service import StopService
 from ovos_core.transformers import MetadataTransformersService, UtteranceTransformersService
 from ovos_plugin_manager.templates.pipeline import IntentMatch
+from ovos_utils.lang import standardize_lang_tag
 from ovos_utils.log import LOG, deprecated, log_deprecation
 from ovos_utils.metrics import Stopwatch
 from padacioso.opm import PadaciosoPipeline as PadaciosoService
@@ -209,7 +210,7 @@ def _load_pipeline_plugins(self):
 
     @property
     def registered_intents(self):
-        lang = get_message_lang()
+        lang = standardize_lang_tag(get_message_lang())
         return [parser.__dict__
                 for parser in self._adapt_service.engines[lang].intent_parsers]
 
@@ -263,7 +264,7 @@ def _handle_transformers(self, message):
         Pipe utterance through transformer plugins to get more metadata.
         Utterances may be modified by any parser and context overwritten
         """
-        lang = get_message_lang(message)  # per query lang or default Configuration lang
+        lang = standardize_lang_tag(get_message_lang())  # per query lang or default Configuration lang
         original = utterances = message.data.get('utterances', [])
         message.context["lang"] = lang
         utterances, message.context = self.utterance_plugins.transform(utterances, message.context)
@@ -281,7 +282,7 @@ def disambiguate_lang(message):
         3 - detected_lang -> tagged by transformers  (text classification, free form chat)
         4 - config lang (or from message.data)
         """
-        default_lang = get_message_lang(message)
+        default_lang = standardize_lang_tag(get_message_lang())
         valid_langs = get_valid_languages()
         lang_keys = ["stt_lang",
                      "request_lang",
@@ -352,6 +353,7 @@ def get_pipeline(self, skips=None, session=None) -> Tuple[str, Callable]:
 
     def _validate_session(self, message, lang):
         # get session
+        lang = standardize_lang_tag(lang)
         sess = SessionManager.get(message)
         if sess.session_id == "default":
             updated = False
@@ -525,7 +527,7 @@ def handle_register_vocab(self, message):
         entity_type = message.data.get('entity_type')
         regex_str = message.data.get('regex')
         alias_of = message.data.get('alias_of')
-        lang = get_message_lang(message)
+        lang = standardize_lang_tag(get_message_lang())
         self._adapt_service.register_vocabulary(entity_value, entity_type,
                                                alias_of, regex_str, lang)
         self.registered_vocab.append(message.data)
@@ -602,7 +604,7 @@ def handle_get_intent(self, message):
             message (Message): message containing utterance
         """
         utterance = message.data["utterance"]
-        lang = get_message_lang(message)
+        lang = standardize_lang_tag(get_message_lang())
         sess = SessionManager.get(message)
 
         # Loop through the matching functions until a match is found.
@@ -653,7 +655,7 @@ def handle_get_adapt(self, message: Message):
             message (Message): message containing utterance
         """
         utterance = message.data["utterance"]
-        lang = get_message_lang(message)
+        lang = standardize_lang_tag(get_message_lang())
         intent = self._adapt_service.match_intent((utterance,), lang, message.serialize())
         intent_data = intent.intent_data if intent else None
         self.bus.emit(message.reply("intent.service.adapt.reply",

ovos_core/intent_services/converse_service.py

@@ -9,6 +9,7 @@
 from ovos_config.locale import setup_locale
 from ovos_plugin_manager.templates.pipeline import IntentMatch, PipelinePlugin
 from ovos_utils import flatten_list
+from ovos_utils.lang import standardize_lang_tag
 from ovos_utils.log import LOG
 from ovos_workshop.permissions import ConverseMode, ConverseActivationMode
 
@@ -279,7 +280,7 @@ def converse(self, utterances, skill_id, lang, message):
             handled (bool): True if handled otherwise False.
         """
         session = SessionManager.get(message)
-        session.lang = lang
+        session.lang = standardize_lang_tag(lang)
 
         state = session.utterance_states.get(skill_id, UtteranceState.INTENT)
         if state == UtteranceState.RESPONSE:
@@ -384,7 +385,7 @@ def handle_deactivate_skill_request(self, message):
 
     def reset_converse(self, message):
         """Let skills know there was a problem with speech recognition"""
-        lang = get_message_lang(message)
+        lang = standardize_lang_tag(get_message_lang())
         try:
             setup_locale(lang)  # restore default lang
         except Exception as e:

ovos_core/intent_services/stop_service.py

@@ -10,6 +10,7 @@
 from ovos_plugin_manager.templates.pipeline import IntentMatch, PipelinePlugin
 from ovos_utils import flatten_list
 from ovos_utils.bracket_expansion import expand_options
+from ovos_utils.lang import standardize_lang_tag
 from ovos_utils.log import LOG
 from ovos_utils.parse import match_one
 
@@ -25,7 +26,7 @@ def __init__(self, bus):
     def load_resource_files(self):
         base = f"{dirname(__file__)}/locale"
         for lang in os.listdir(base):
-            lang2 = lang.split("-")[0].lower()
+            lang2 = standardize_lang_tag(lang)
             self._voc_cache[lang2] = {}
             for f in os.listdir(f"{base}/{lang}"):
                 with open(f"{base}/{lang}/{f}", encoding="utf-8") as fi:
@@ -127,7 +128,7 @@ def match_stop_high(self, utterances: List[str], lang: str, message: Message) ->
         Returns:
             IntentMatch if handled otherwise None.
         """
-        lang = lang.split("-")[0]
+        lang = standardize_lang_tag(lang)
         if lang not in self._voc_cache:
             return None
 
@@ -178,7 +179,7 @@ def match_stop_medium(self, utterances: List[str], lang: str, message: Message)
         Returns:
             IntentMatch if handled otherwise None.
         """
-        lang = lang.split("-")[0]
+        lang = standardize_lang_tag(lang)
         if lang not in self._voc_cache:
             return None
 
@@ -205,7 +206,7 @@ def match_stop_low(self, utterances: List[str], lang: str, message: Message) ->
         Returns:
             IntentMatch if handled otherwise None.
         """
-        lang = lang.split("-")[0]
+        lang = standardize_lang_tag(lang)
         if lang not in self._voc_cache:
             return None
         sess = SessionManager.get(message)
@@ -266,7 +267,7 @@ def voc_match(self, utt: str, voc_filename: str, lang: str,
         Returns:
             bool: True if the utterance has the given vocabulary it
         """
-        lang = lang.split("-")[0].lower()
+        lang = standardize_lang_tag(lang)
         if lang not in self._voc_cache:
             return False
 

requirements/lgpl.txt

@@ -1,2 +1,2 @@
-ovos_padatious>=0.1.2,<1.0.0
+ovos_padatious>=0.1.3,<1.0.0
 fann2>=1.0.7, < 1.1.0

requirements/requirements.txt

@@ -3,13 +3,13 @@ python-dateutil>=2.6, <3.0
 watchdog>=2.1, <3.0
 combo-lock>=0.2.2, <0.3
 
-padacioso>=0.2.2,<1.0.0
-ovos-adapt-parser>=0.1.2, <1.0.0
-ovos_ocp_pipeline_plugin>=0.1.2, <1.0.0
-ovos-common-query-pipeline-plugin>=0.1.2, <1.0.0
+padacioso>=0.2.4,<1.0.0
+ovos-adapt-parser>=0.1.3, <1.0.0
+ovos_ocp_pipeline_plugin>=0.1.3, <1.0.0
+ovos-common-query-pipeline-plugin>=0.1.4, <1.0.0
 
-ovos-utils>=0.1.0,<1.0.0
-ovos_bus_client>=0.1.0,<1.0.0
+ovos-utils>=0.3.5,<1.0.0
+ovos_bus_client>=0.1.4,<1.0.0
 ovos-plugin-manager>=0.0.26,<1.0.0
 ovos-config>=0.0.13,<1.0.0
 ovos-lingua-franca>=0.4.7,<1.0.0

test/end2end/routing/test_sched.py

@@ -16,7 +16,7 @@ def setUp(self):
     def test_no_session(self):
         SessionManager.sessions = {}
         SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
-        SessionManager.default_session.lang = "en-us"
+        SessionManager.default_session.lang = "en-US"
         SessionManager.pipeline = ["adapt_high"]
 
         messages = []

test/end2end/routing/test_session.py

@@ -21,7 +21,7 @@ def tearDown(self) -> None:
     def test_no_session(self):
         SessionManager.sessions = {}
         SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
-        SessionManager.default_session.lang = "en-us"
+        SessionManager.default_session.lang = "en-US"
         SessionManager.pipeline = ["adapt_high"]
 
         messages = []

test/end2end/session/test_blacklist.py

@@ -21,7 +21,7 @@ def tearDown(self) -> None:
     def test_blacklist(self):
         SessionManager.sessions = {}
         SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
-        SessionManager.default_session.lang = "en-us"
+        SessionManager.default_session.lang = "en-US"
         SessionManager.default_session.pipeline = ["adapt_high"]
         SessionManager.default_session.blacklisted_skills = []
         SessionManager.default_session.blacklisted_intents = []
@@ -309,7 +309,7 @@ def tearDown(self) -> None:
     def test_fallback(self):
         SessionManager.sessions = {}
         SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
-        SessionManager.default_session.lang = "en-us"
+        SessionManager.default_session.lang = "en-US"
         SessionManager.default_session.pipeline = [
             "fallback_high"
         ]
@@ -400,7 +400,7 @@ def tearDown(self) -> None:
     def test_common_qa(self):
         SessionManager.sessions = {}
         SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
-        SessionManager.default_session.lang = "en-us"
+        SessionManager.default_session.lang = "en-US"
         SessionManager.default_session.pipeline = ["common_qa"]
         messages = []
 

test/end2end/session/test_complete_failure.py

@@ -19,7 +19,7 @@ def tearDown(self) -> None:
     def test_complete_failure(self):
         SessionManager.sessions = {}
         SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
-        SessionManager.default_session.lang = "en-us"
+        SessionManager.default_session.lang = "en-US"
         SessionManager.default_session.active_skills = [(self.skill_id, time.time())]
         SessionManager.default_session.pipeline = [
                            "stop_high",
@@ -105,7 +105,7 @@ def wait_for_n_messages(n):
     def test_complete_failure_lang_detect(self):
         SessionManager.sessions = {}
         SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
-        SessionManager.default_session.lang = "en-us"
+        SessionManager.default_session.lang = "en-US"
         SessionManager.default_session.active_skills = [(self.skill_id, time.time())]
         SessionManager.default_session.pipeline = [
                            "stop_high",
@@ -144,7 +144,7 @@ def wait_for_n_messages(n):
 
         self.core.bus.on("message", new_msg)
 
-        SessionManager.default_session.valid_languages = ["en-us", stt_lang_detect, "fr-fr"]
+        SessionManager.default_session.valid_languages = ["en-US", stt_lang_detect, "fr-fr"]
         utt = Message("recognizer_loop:utterance",
                       {"utterances": ["hello world"]},
                       {"session": SessionManager.default_session.serialize(),
@@ -247,4 +247,4 @@ def wait_for_n_messages(n):
         self.assertEqual(messages[18].data["session_data"]["lang"], "pt-pt")
         self.assertEqual(SessionManager.default_session.lang, "pt-pt")
 
-        SessionManager.default_session.lang = "en-us"
+        SessionManager.default_session.lang = "en-US"

test/end2end/session/test_converse.py

@@ -21,7 +21,7 @@ def tearDown(self) -> None:
     def test_no_session(self):
         SessionManager.sessions = {}
         SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
-        SessionManager.default_session.lang = "en-us"
+        SessionManager.default_session.lang = "en-US"
         SessionManager.default_session.pipeline = [
             "converse",
             "padatious_high",
@@ -85,7 +85,7 @@ def wait_for_n_messages(n):
         # (missing in utterance message) and kept in all messages
         for m in messages[1:]:
             self.assertEqual(m.context["session"]["session_id"], "default")
-            self.assertEqual(m.context["lang"], "en-us")
+            self.assertEqual(m.context["lang"], "en-US")
 
         # verify skill is activated
         self.assertEqual(messages[1].msg_type, "intent.service.skills.activated")
@@ -157,7 +157,7 @@ def wait_for_n_messages(n):
         # (missing in utterance message) and kept in all messages
         for m in messages[1:]:
             self.assertEqual(m.context["session"]["session_id"], "default")
-            self.assertEqual(m.context["lang"], "en-us")
+            self.assertEqual(m.context["lang"], "en-US")
 
         # verify that "lang" is injected by converse.ping
         # (missing in utterance message) and kept in all messages
@@ -250,7 +250,7 @@ def wait_for_n_messages(n):
         # (missing in utterance message) and kept in all messages
         for m in messages[1:]:
             self.assertEqual(m.context["session"]["session_id"], "default")
-            self.assertEqual(m.context["lang"], "en-us")
+            self.assertEqual(m.context["lang"], "en-US")
 
         # converse
         self.assertEqual(messages[1].msg_type, f"{self.other_skill_id}.converse.ping")
@@ -339,7 +339,7 @@ def wait_for_n_messages(n):
         # (missing in utterance message) and kept in all messages
         for m in messages[1:]:
             self.assertEqual(m.context["session"]["session_id"], "default")
-            self.assertEqual(m.context["lang"], "en-us")
+            self.assertEqual(m.context["lang"], "en-US")
 
         # converse
         self.assertEqual(messages[1].msg_type, f"{self.skill_id}.converse.ping")

test/end2end/session/test_fallback.py

@@ -19,7 +19,7 @@ def tearDown(self) -> None:
     def test_fallback(self):
         SessionManager.sessions = {}
         SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
-        SessionManager.default_session.lang = "en-us"
+        SessionManager.default_session.lang = "en-US"
         SessionManager.default_session.pipeline = [
             "converse",
             "fallback_high",
@@ -142,7 +142,7 @@ def wait_for_n_messages(n):
     def test_fallback_with_session(self):
         SessionManager.sessions = {}
         SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
-        SessionManager.default_session.lang = "en-us"
+        SessionManager.default_session.lang = "en-US"
         SessionManager.default_session.pipeline = [
             "fallback_high",
             "fallback_medium",
@@ -316,7 +316,7 @@ def tearDown(self) -> None:
     def test_fallback(self):
         SessionManager.sessions = {}
         SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
-        SessionManager.default_session.lang = "en-us"
+        SessionManager.default_session.lang = "en-US"
         SessionManager.default_session.pipeline = [
             "fallback_medium",
             "fallback_low"