init

Author: dhohirpradana

contents.json

@@ -0,0 +1,50 @@
+# youtube search scraper
+import requests
+import json
+from bs4 import BeautifulSoup
+import re
+
+response = requests.get(
+    "https://www.youtube.com/results?search_query=indonesia").text
+
+soup = BeautifulSoup(response, 'lxml')
+
+try:
+    script = soup.find_all('script', text=re.compile('ytInitialData'))
+    json_text = re.search(
+        r'ytInitialData = ({.*?});', script[0].string, re.DOTALL).group(1)
+    with open('data.json', 'w', encoding="utf-8") as outfile:
+        outfile.write(json_text)
+    json_data = json.loads(json_text)
+    contents = json_data['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents']
+    with open('contents.json', 'w', encoding="utf-8") as outfile:
+        outfile.write(json.dumps(contents))
+    for i, content in enumerate(contents):
+        if 'itemSectionRenderer' in content:
+            for item in content['itemSectionRenderer']['contents']:
+                if 'videoRenderer' in item:
+                    video = item['videoRenderer']
+                    title = video['title']['runs'][0]['text']
+                    video_id = video['videoId']
+                    published_time = video['publishedTimeText']['simpleText']
+                    view_count = video['viewCountText']['simpleText']
+                    obj = {
+                        "title": title,
+                        "video_id": video_id,
+                        "published_time": published_time,
+                        "view_count": view_count,
+                    }
+                    print("Video:", obj)
+                elif 'playlistRenderer' in item:
+                    playlist = item['playlistRenderer']
+                    title = playlist['title']['simpleText']
+                    playlist_id = playlist['playlistId']
+                    video_count = playlist['videoCount']
+                    obj = {
+                        "title": title,
+                        "playlist_id": playlist_id,
+                        "video_count": video_count,
+                    }
+                    print("Playlist:", obj)
+except Exception as e:
+    print(str(e))

data.json

@@ -0,0 +1,22 @@
+import time
+
+from selenium import webdriver
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.common.by import By
+
+browser = webdriver.Chrome()
+
+browser.get("https://www.youtube.com/results?search_query=indonesia")
+time.sleep(1)
+
+print(browser.find_elements(By.XPATH, "//div[@id='contents']"))
+
+# no_of_pagedowns = 20
+
+# while no_of_pagedowns:
+#     elem.send_keys(Keys.PAGE_DOWN)
+#     time.sleep(0.2)
+#     no_of_pagedowns-=1
+
+# for post in post_elems:
+#     print(post.text)