[pull] master from scrapinghub:master

dateparser/search/search.py

@@ -34,15 +34,13 @@ def _add_time_span_results(results, text, settings):
 class _ExactLanguageSearch:
     def __init__(self, loader):
         self.loader = loader
-        self.language = None
 
     def get_current_language(self, shortname):
-        if self.language is None or self.language.shortname != shortname:
-            self.language = self.loader.get_locale(shortname)
+        return self.loader.get_locale(shortname)
 
     def search(self, shortname, text, settings):
-        self.get_current_language(shortname)
-        result = self.language.translate_search(text, settings=settings)
+        language = self.get_current_language(shortname)
+        result = language.translate_search(text, settings=settings)
         return result
 
     @staticmethod
@@ -270,13 +268,13 @@ def detect_language(
             )
 
         if languages:
-            self.language_detector = FullTextLanguageDetector(languages=languages)
+            language_detector = FullTextLanguageDetector(languages=languages)
         else:
-            self.language_detector = FullTextLanguageDetector(
+            language_detector = FullTextLanguageDetector(
                 list(self.available_language_map.values())
             )
 
-        detected_language = self.language_detector._best_language(text) or (
+        detected_language = language_detector._best_language(text) or (
             settings.DEFAULT_LANGUAGES[0] if settings.DEFAULT_LANGUAGES else None
         )
         return detected_language

tests/test_thread_safety.py

@@ -8,14 +8,16 @@
 from dateparser.date import DateDataParser
 from dateparser.languages.dictionary import Dictionary
 from dateparser.search import search_dates
+from dateparser.search.search import DateSearchWithDetection, _ExactLanguageSearch
 from tests import BaseTestCase
 
 
 class TestThreadSafety(BaseTestCase):
     """Regression tests for thread-safety issues.
 
-    See https://github.com/scrapinghub/dateparser/issues/441 and
-    https://github.com/scrapinghub/dateparser/issues/1291.
+    See https://github.com/scrapinghub/dateparser/issues/441,
+    https://github.com/scrapinghub/dateparser/issues/1291 and
+    https://github.com/scrapinghub/dateparser/issues/1369.
     """
 
     def setUp(self):
@@ -123,6 +125,97 @@ def search(i):
         self.assertTrue(all(result is not None for result in results))
         self.assertEqual(observed, {baseline})
 
+    def test_detect_language_does_not_leave_narrowed_detector_on_instance(self):
+        # Issue #1369 site 1: detect_language used to stash a FullTextLanguageDetector
+        # on the process-wide singleton. _best_language narrows detector.languages
+        # in place, so a concurrent search_dates can load another call's already-
+        # narrowed detector and return None or a wrong date.
+        ds = DateSearchWithDetection()
+        detected = ds.detect_language("19 марта 2001", languages=["ru", "en"])
+        self.assertEqual(detected, "ru")
+        detector = getattr(ds, "language_detector", None)
+        if detector is not None:
+            leftover = [locale.shortname for locale in detector.languages]
+            self.assertEqual(
+                set(leftover),
+                {"ru", "en"},
+                "detect_language left a single-use detector narrowed in place "
+                "on the instance; concurrent search_dates can observe this "
+                "(issue #1369). leftover=%r" % leftover,
+            )
+
+    def test_concurrent_search_dates_does_not_share_language_detector(self):
+        # Issue #1369 site 1: the store/load gap is a few bytecodes, so park each
+        # thread immediately after it stores the detector. Sequential search_dates
+        # is not a valid RED for this race.
+        ru_text = "Договор подписан 19 марта 2001 года в Москве"
+        en_text = "The satellite was launched on 4 October 1957 from Baikonur"
+        sequential_ru = search_dates(ru_text, languages=["ru"])
+        sequential_en = search_dates(en_text, languages=["en"])
+        self.assertIsNotNone(sequential_ru)
+        self.assertIsNotNone(sequential_en)
+
+        slot = {}
+        barrier = threading.Barrier(2, timeout=10)
+
+        def _get(self):
+            return slot["v"]
+
+        def _set(self, value):
+            slot["v"] = value
+            try:
+                barrier.wait()
+            except threading.BrokenBarrierError:
+                pass
+
+        DateSearchWithDetection.language_detector = property(_get, _set)
+        try:
+            with ThreadPoolExecutor(max_workers=2) as executor:
+                ru, en = executor.map(
+                    lambda args: search_dates(args[0], languages=[args[1]]),
+                    [(ru_text, "ru"), (en_text, "en")],
+                )
+        finally:
+            del DateSearchWithDetection.language_detector
+
+        self.assertEqual(ru, sequential_ru)
+        self.assertEqual(en, sequential_en)
+
+    def test_concurrent_search_dates_does_not_share_locale_slot(self):
+        # Issue #1369 site 2: _ExactLanguageSearch.self.language is a 1-slot cache
+        # on the same singleton. Park after get_current_language so a store from
+        # the other thread lands before translate_search.
+        ru_text = "Договор подписан 19 марта 2001 года в Москве"
+        en_text = "The satellite was launched on 4 October 1957 from Baikonur"
+        sequential_ru = search_dates(ru_text, languages=["ru"])
+        sequential_en = search_dates(en_text, languages=["en"])
+        self.assertIsNotNone(sequential_ru)
+        self.assertIsNotNone(sequential_en)
+
+        original = _ExactLanguageSearch.get_current_language
+        barrier = threading.Barrier(2, timeout=10)
+
+        def patched(self, shortname):
+            result = original(self, shortname)
+            try:
+                barrier.wait()
+            except threading.BrokenBarrierError:
+                pass
+            return result
+
+        _ExactLanguageSearch.get_current_language = patched
+        try:
+            with ThreadPoolExecutor(max_workers=2) as executor:
+                ru, en = executor.map(
+                    lambda args: search_dates(args[0], languages=[args[1]]),
+                    [(ru_text, "ru"), (en_text, "en")],
+                )
+        finally:
+            _ExactLanguageSearch.get_current_language = original
+
+        self.assertEqual(ru, sequential_ru)
+        self.assertEqual(en, sequential_en)
+
 
 if __name__ == "__main__":
     unittest.main()