diff --git a/config/languages/dutch.php b/config/languages/dutch.php new file mode 100644 index 0000000..69499f7 --- /dev/null +++ b/config/languages/dutch.php @@ -0,0 +1,199 @@ + [ + 'mild' => [ + 'stom', 'stomme', 'stomkop', 'dom', 'domme', 'domkop', + 'sukkel', 'sul', 'oen', 'sufferd', 'kluns', 'laf', 'laffe', + 'idioot', 'stomzinnig', 'stommeling', + ], + 'moderate' => [ + 'klootzak', 'eikel', 'lulhannes', 'zak', 'zakkenroller', + 'godverdomme', 'godver', 'verdomme', 'verdorie', 'verdomd', + 'slet', 'hoer', 'trut', 'teef', 'kreng', 'wijf', + 'schoft', 'rotzak', 'vuil', 'vuile', + 'debiel', 'mongool', 'imbeciel', 'tieten', + ], + 'high' => [ + 'kut', 'lul', 'neuken', 'neuk', 'geneukt', + 'schijt', 'schijten', 'schijter', + 'tering', 'kanker', 'tyfus', 'cholera', 'pest', + 'klere', 'klote', 'kloten', 'moederneuker', + 'poepen', 'poep', 'pissen', 'pis', 'kosts', + 'ballen', 'pik', 'kontgat', 'pijpen', 'piemel', 'rukken', 'aftrekken' + ], + 'extreme' => [ + 'nikker', 'nigger', 'spleetoog', 'flikker', 'kankerjood', 'kankerneger', 'kankermongool', + 'tyfusmongool', 'teringlijer', 'kankerhomo', + ], + ], + + 'profanities' => [ + 'ballen', + 'debieltje', + 'debiel', + 'dom', + 'dombo', + 'domkop', + 'domme', + 'dommerd', + 'dommerik', + 'domoor', + 'eikel', + 'eikelbak', + 'flikker', + 'flikkerig', + 'godver', + 'godverdomme', + 'godverdorie', + 'hoer', + 'hoerenjong', + 'hoerenkind', + 'hoerenloper', + 'hoerenzoon', + 'imbeciel', + 'kanker', + 'kankerbak', + 'kankerlijer', + 'kankermongool', + 'kankerneger', + 'kankerpatiënt', + 'klere', + 'klerelijer', + 'kloot', + 'klootzak', + 'klote', + 'kloten', + 'kluns', + 'kon', + 'kont', + 'kontgat', + 'konthaar', + 'kontzak', + 'kut', + 'kutding', + 'kuthoer', + 'kutkind', + 'kutterig', + 'kutwijk', + 'kutwijf', + 'laf', + 'laffe', + 'lul', + 'lulhannes', + 'lullo', + 'lulverhaal', + 'mongool', + 'mongoolse', + 'neuk', + 'neuken', + 'neuker', + 'oen', + 'pest', + 'pestlijer', + 'pik', + 'pikhaar', + 'pis', + 'pisgat', + 'pissen', + 'pisser', + 'poep', + 'poeperd', + 'poepen', + 'reet', + 'reetlikker', + 'rot', + 'rotzak', + 'schijt', + 'schijtebroek', + 'schijten', + 'schijter', + 'schoft', + 'slet', + 'sletje', + 'sloerie', + 'sloeries', + 'stom', + 'stommeling', + 'stomme', + 'stommerik', + 'stomkop', + 'stomzinnig', + 'sufferd', + 'sukkeltje', + 'sukkel', + 'sul', + 'tering', + 'teringlijer', + 'teringlijder', + 'trut', + 'truttig', + 'tyfus', + 'tyfuslijer', + 'tyfuslijder', + 'verdomme', + 'verdorie', + 'verdomd', + 'vuil', + 'vuile', + 'vuilak', + 'wijf', + 'wijfje', + 'zak', + 'zakkenroller', + 'zwijn', + 'zwijntje', + 'teef', + 'kreng', + 'debielenbak', + 'godverd', + 'cholera', + 'choleralijder', + 'pokken', + 'pokkelijer', + ], + + 'false_positives' => [ + 'scoutskamp', + 'piket', + 'pikant', + 'pikanterie', + 'pistool', + 'pistolet', + 'pistache', + 'episode', + 'rots', + 'rotsen', + 'rotatie', + 'router', + 'domein', + 'domino', + 'dominee', + 'document', + 'random', + 'pesticide', + 'besproeien', + 'zakelijk', + 'zakenman', + 'zakken', + 'zakdoek', + 'slaapzak', + 'rugzak', + 'werkzak', + 'broekzak', + 'kankeronderzoek', + 'kankerbehandeling', + 'antikanker', + 'reetje', + 'domburg', + 'rotterdam', + ], + + 'substitutions' => [ + '/e/' => ['e', '3', '€', 'ë', 'é', 'è', 'ê', 'ē', 'ė', 'ę', '∑'], + '/i/' => ['i', '!', '|', '1', 'ï', 'î', 'í', 'ì', 'ī', 'į'], + '/o/' => ['o', '0', 'ö', 'ô', 'ó', 'ò', 'ø', 'ō', 'õ'], + '/u/' => ['u', 'ü', 'û', 'ú', 'ù', 'ū'], + '/a/' => ['a', '4', '@', 'á', 'à', 'â', 'ä', 'ã', 'å'], + ], +]; diff --git a/src/Core/Dictionary.php b/src/Core/Dictionary.php index 4f95b40..8923c59 100644 --- a/src/Core/Dictionary.php +++ b/src/Core/Dictionary.php @@ -8,6 +8,7 @@ use Blaspsoft\Blasp\Core\Normalizers\EnglishNormalizer; use Blaspsoft\Blasp\Core\Normalizers\SpanishNormalizer; use Blaspsoft\Blasp\Core\Normalizers\GermanNormalizer; +use Blaspsoft\Blasp\Core\Normalizers\DutchNormalizer; use Blaspsoft\Blasp\Core\Normalizers\FrenchNormalizer; use Illuminate\Support\Facades\Cache; @@ -325,6 +326,7 @@ public static function getNormalizerForLanguage(string $language): StringNormali 'spanish' => new SpanishNormalizer(), 'german' => new GermanNormalizer(), 'french' => new FrenchNormalizer(), + 'dutch' => new DutchNormalizer(), default => new EnglishNormalizer(), }; } diff --git a/src/Core/Normalizers/DutchNormalizer.php b/src/Core/Normalizers/DutchNormalizer.php new file mode 100644 index 0000000..3512b52 --- /dev/null +++ b/src/Core/Normalizers/DutchNormalizer.php @@ -0,0 +1,28 @@ + 'e', 'Ë' => 'E', + 'ï' => 'i', 'Ï' => 'I', + 'ü' => 'u', 'Ü' => 'U', + 'ö' => 'o', 'Ö' => 'O', + 'é' => 'e', 'É' => 'E', + 'è' => 'e', 'È' => 'E', + 'ê' => 'e', 'Ê' => 'E', + 'á' => 'a', 'Á' => 'A', + 'à' => 'a', 'À' => 'A', + 'â' => 'a', 'Â' => 'A', + 'ó' => 'o', 'Ó' => 'O', + 'ú' => 'u', 'Ú' => 'U', + 'û' => 'u', 'Û' => 'U', + 'í' => 'i', 'Í' => 'I', + ]; + + return strtr($string, $dutchMappings); + } +} diff --git a/src/PendingCheck.php b/src/PendingCheck.php index 97f20a4..d0c61d2 100644 --- a/src/PendingCheck.php +++ b/src/PendingCheck.php @@ -146,6 +146,11 @@ public function french(): self return $this->in('french'); } + public function dutch(): self + { + return $this->in('dutch'); + } + // --- Configure (backward-compat) --- public function configure(?array $profanities = null, ?array $falsePositives = null): self diff --git a/tests/AllLanguagesApiTest.php b/tests/AllLanguagesApiTest.php index 4e1348c..db85fe4 100644 --- a/tests/AllLanguagesApiTest.php +++ b/tests/AllLanguagesApiTest.php @@ -27,10 +27,10 @@ public function test_all_languages_detection() public function test_mixed_language_content() { - $result = Blasp::allLanguages()->check('This shit is mierda and scheiße'); + $result = Blasp::allLanguages()->check('This shit is mierda, kut and scheiße'); $this->assertTrue($result->hasProfanity()); - $this->assertEquals('This **** is ****** and *******', $result->getCleanString()); - $this->assertEquals(3, $result->getProfanitiesCount()); + $this->assertEquals('This **** is ******, *** and *******', $result->getCleanString()); + $this->assertEquals(4, $result->getProfanitiesCount()); } public function test_chainable_all_languages() @@ -41,17 +41,18 @@ public function test_chainable_all_languages() public function test_language_shortcuts_vs_all() { - $text = 'fucking merde scheiße mierda'; + $text = 'fucking merde scheiße mierda godverdomme'; $englishResult = Blasp::english()->check($text); $this->assertEquals(1, $englishResult->getProfanitiesCount()); $allResult = Blasp::allLanguages()->check($text); - $this->assertEquals(4, $allResult->getProfanitiesCount()); + $this->assertEquals(5, $allResult->getProfanitiesCount()); $this->assertStringNotContainsString('fucking', $allResult->getCleanString()); $this->assertStringNotContainsString('merde', $allResult->getCleanString()); $this->assertStringNotContainsString('scheiße', $allResult->getCleanString()); + $this->assertStringNotContainsString('godverdomme', $allResult->getCleanString()); $this->assertStringContainsString('*******', $allResult->getCleanString()); } diff --git a/tests/DutchLanguageDetectionTest.php b/tests/DutchLanguageDetectionTest.php new file mode 100644 index 0000000..49bc007 --- /dev/null +++ b/tests/DutchLanguageDetectionTest.php @@ -0,0 +1,140 @@ +check('wat een kut dag'); + $this->assertTrue($result->isOffensive()); + } + + public function test_detects_multiple_dutch_profanities() + { + $result = Blasp::dutch()->check('godverdomme wat een klootzak'); + $this->assertTrue($result->isOffensive()); + $this->assertGreaterThanOrEqual(2, $result->count()); + } + + public function test_clean_text_is_not_offensive() + { + $result = Blasp::dutch()->check('Goedemorgen, hoe gaat het met jou?'); + $this->assertFalse($result->isOffensive()); + } + + public function test_censors_dutch_profanity() + { + $result = Blasp::dutch()->check('wat een kut dag'); + $this->assertStringContainsString('*', $result->clean()); + $this->assertStringNotContainsString('kut', strtolower($result->clean())); + } + + public function test_detects_mild_severity() + { + $result = Blasp::dutch()->check('wat een stomme fout'); + $this->assertTrue($result->isOffensive()); + } + + public function test_detects_moderate_severity() + { + $result = Blasp::dutch()->check('jij bent een eikel'); + $this->assertTrue($result->isOffensive()); + } + + public function test_detects_high_severity() + { + $result = Blasp::dutch()->check('dit is echt kut'); + $this->assertTrue($result->isOffensive()); + } + + public function test_detects_disease_curses() + { + $result = Blasp::dutch()->check('tering wat een dag'); + $this->assertTrue($result->isOffensive()); + + $result = Blasp::dutch()->check('tyfus wat een rotdag'); + $this->assertTrue($result->isOffensive()); + } + + public function test_detects_case_insensitive() + { + $variants = ['KUT', 'Kut', 'kut', 'KuT']; + + foreach ($variants as $variant) { + $result = Blasp::dutch()->check("wat een $variant dag"); + $this->assertTrue($result->isOffensive(), "Failed to detect '$variant'"); + } + } + + public function test_detects_profanity_with_diacritics() + { + // Obfuscation using diacritics should still be detected + $result = Blasp::dutch()->check('wat een küt dag'); + $this->assertTrue($result->isOffensive(), 'Failed to detect diacritic obfuscation of kut'); + } + + public function test_false_positive_zakelijk() + { + $result = Blasp::dutch()->check('Dit is een zakelijk gesprek'); + $this->assertFalse($result->isOffensive()); + } + + public function test_false_positive_document() + { + $result = Blasp::dutch()->check('Heb je het document al gelezen?'); + $this->assertFalse($result->isOffensive()); + } + + public function test_false_positive_domein() + { + $result = Blasp::dutch()->check('Het domein is geregistreerd'); + $this->assertFalse($result->isOffensive()); + } + + public function test_false_positive_pistool() + { + $result = Blasp::dutch()->check('De politie droeg een pistool'); + $this->assertFalse($result->isOffensive()); + } + + public function test_false_positive_rots() + { + $result = Blasp::dutch()->check('We klommen op de rots'); + $this->assertFalse($result->isOffensive()); + } + + public function test_sentence_with_mixed_clean_and_profane_words() + { + $result = Blasp::dutch()->check('Ik heb een kut dag gehad op het werk'); + $this->assertTrue($result->isOffensive()); + $clean = $result->clean(); + $this->assertStringContainsString('dag', $clean); + $this->assertStringContainsString('werk', $clean); + $this->assertStringNotContainsString('kut', strtolower($clean)); + } + + public function test_empty_string_is_not_offensive() + { + $result = Blasp::dutch()->check(''); + $this->assertFalse($result->isOffensive()); + } + + public function test_dutch_language_available() + { + $languages = \Blaspsoft\Blasp\Core\Dictionary::getAvailableLanguages(); + $this->assertContains('dutch', $languages); + } + + public function test_in_dutch_is_equivalent_to_dutch_shorthand() + { + $text = 'wat een kut dag'; + $result1 = Blasp::in('dutch')->check($text); + $result2 = Blasp::dutch()->check($text); + + $this->assertEquals($result1->isOffensive(), $result2->isOffensive()); + $this->assertEquals($result1->count(), $result2->count()); + } +} diff --git a/tests/DutchStringNormalizerTest.php b/tests/DutchStringNormalizerTest.php new file mode 100644 index 0000000..65043b0 --- /dev/null +++ b/tests/DutchStringNormalizerTest.php @@ -0,0 +1,55 @@ +normalizer = new DutchNormalizer(); + } + + public function test_normalize_dutch_profanity_variants() + { + // kut with trema obfuscation + $this->assertEquals('kut', $this->normalizer->normalize('küt')); + $this->assertEquals('kut', $this->normalizer->normalize('kût')); + $this->assertEquals('lul', $this->normalizer->normalize('lül')); + $this->assertEquals('neuken', $this->normalizer->normalize('neüken')); + $this->assertEquals('schijt', $this->normalizer->normalize('schïjt')); + } + + public function test_normalize_common_dutch_words_with_diacritics() + { + $this->assertEquals('een', $this->normalizer->normalize('één')); + $this->assertEquals('meteen', $this->normalizer->normalize('metëen')); + $this->assertEquals('voor', $this->normalizer->normalize('vóór')); + } + + public function test_normalize_mixed_case_preservation() + { + $this->assertEquals('KUT', $this->normalizer->normalize('KÜT')); + $this->assertEquals('Lul', $this->normalizer->normalize('Lül')); + $this->assertEquals('NEUKEN', $this->normalizer->normalize('NEÜKEN')); + } + + public function test_normalize_dutch_profanities_from_config() + { + $config = require __DIR__ . '/../config/languages/dutch.php'; + $profanities = array_slice($config['profanities'], 0, 20); + + foreach ($profanities as $profanity) { + $normalized = $this->normalizer->normalize($profanity); + $this->assertDoesNotMatchRegularExpression( + '/[àâäáèéêëìíîïòóôöùúûüÀÂÄÁÈÉÊËÌÍÎÏÒÓÔÖÙÚÛÜ]/', + $normalized, + "Dutch profanity '$profanity' still contains diacritics after normalization: '$normalized'" + ); + } + } +}