Merge pull request #544 from osapon/master

(fix): Improved recognition of text encoding

Author: oscarotero

.gitattributes

@@ -7,3 +7,5 @@
 .php_cs.dist      export-ignore
 .travis.yml       export-ignore
 phpunit.xml.dist  export-ignore
+/tests/cache/4pda.to.2022-12-04-406834-sostoyalsya_reliz_clown_of_duty_parodii_na_call_of_duty.php working-tree-encoding=windows-1251 diff=windows-1251
+/tests/cache/www.itmedia.co.jp.news-articles-2410-28-news159.html.php working-tree-encoding=sjis diff=sjis

src/Document.php

@@ -26,7 +26,18 @@ public function __construct(Extractor $extractor)
         $html = str_replace('<br>', "\n<br>", $html);
         $html = str_replace('<br ', "\n<br ", $html);
 
-        $this->document = !empty($html) ? Parser::parse($html) : new DOMDocument();
+        $encoding = null;
+        $contentType = $extractor->getResponse()->getHeaderLine('content-type');
+        preg_match('/charset="?(.*?)(?=$|\s|;|")/i', $contentType, $match);
+        if (!empty($match[1])) {
+            $encoding = $match[1];
+        } elseif (!empty($html)) {
+            preg_match('/charset="?(.*?)(?=$|\s|;|")/i', $html, $match);
+            if (!empty($match[1])) {
+                $encoding = $match[1];
+            }
+        }
+        $this->document = !empty($html) ? Parser::parse($html, $encoding) : new DOMDocument();
         $this->initXPath();
     }
 

tests/PagesTest.php

@@ -52,6 +52,8 @@ public function urlDataProvider(): array
             ['http://www.ustream.tv/channel/red-shoes-billiards-60803-camera-1'],
             ['http://www.viddler.com/v/bdce8c7'],
             ['http://www.wired.com/?p=2064839'],
+            ['https://www.itmedia.co.jp/news/articles/2410/28/news159.html'],
+            ['https://4pda.to/2022/12/04/406834/sostoyalsya_reliz_clown_of_duty_parodii_na_call_of_duty/'],
         ];
     }