flask

Author: dhohirpradana

.env

@@ -0,0 +1,15 @@
+# CLUSTER_API_ENDPOINT=https://10.10.72.54/v3/clusters
+# BEARER_TOKEN=token-lcp92:bwvvzk4cfzldvchzh9kk6pm8zpz7xrbtbpszxmj6rbmxsjl4b88dkl
+# NAMESPACE=tester
+# CLUSTER=c-rxtpn
+# MICROGEN_USER_URL=https://dev-khayanganjxhpv.microgen.id/api/user
+# GENERATE_USER=https://ujirnnbxgq.function.microgen.id/api/v1/generate/user
+# USER_BLOCKCHAIN_URL=https://dev-khayanganjxhpv.microgen.id/api/userBlockChains
+
+CLUSTER_API_ENDPOINT=https://10.10.72.89/v3/clusters
+BEARER_TOKEN=token-vbqz2:drkj26x7d4f9rwx27cdmsthn5nkpjvdzl8rlhn92pxhdhzd2r9ccqd
+NAMESPACE=blockchain
+CLUSTER=c-zmj9v
+MICROGEN_USER_URL=https://dev-khayanganjxhpv.microgen.id/api/user
+GENERATE_USER=https://ujirnnbxgq.function.microgen.id/api/v1/generate/user
+USER_BLOCKCHAIN_URL=https://dev-khayanganjxhpv.microgen.id/api/userBlockChains

.gitignore

@@ -0,0 +1,2 @@
+# .env
+__pycache__

Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.8-slim-buster
+
+WORKDIR /app
+COPY requirements.txt requirements.txt
+RUN pip3 install -r requirements.txt
+COPY . .
+
+ENV FLASK_RUN_HOST=0.0.0.0
+
+CMD ["python3", "-m", "flask", "run"]

README copy.md

@@ -0,0 +1 @@
+# khayangan-blockchain

app.py

@@ -0,0 +1,21 @@
+import os
+from sys import stderr
+
+from flask import Flask, jsonify, request
+from flask_cors import CORS
+
+from yt_scraper_sroll import handler as yt_scraper_sroll_handler
+
+app = Flask(__name__)
+CORS(app, resources={r"/*": {"origins": "*"}})
+
+@app.route('/')
+def hello_geek():
+    return '<h1>Hello from Flask</h2>'
+
+@app.route('/youtube_scraper_scroll', methods=['POST'])
+def youtube_scraper_scroll():
+    return yt_scraper_sroll_handler(request, jsonify)
+
+if __name__ == "__main__":
+    app.run(debug=True)

geckodriver.log

@@ -0,0 +1,13 @@
+certifi==2022.6.15
+charset-normalizer==2.1.1
+click==8.0.3
+Flask==2.0.2
+idna==3.3
+itsdangerous==2.0.1
+Jinja2==3.0.2
+MarkupSafe==2.0.1
+requests==2.28.1
+urllib3==1.26.12
+Werkzeug==2.0.2
+python-dotenv==0.21.0
+flask_cors==3.0.3

requirements.txt

@@ -4,6 +4,7 @@
 from selenium.webdriver.support.wait import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 import urllib.parse
+import os
 import datetime
 
 # options = webdriver.ChromeOptions()
@@ -36,7 +37,7 @@
 video_views = []
 video_published_times = []
 
-max_scroll = 10
+max_scroll = 2
 file_name = f"{query}_scroll-{max_scroll}_{now.strftime('%Y%m%d_%H%M%S')}"
 # while True:
 while max_scroll > 0:
@@ -46,7 +47,7 @@
     for i, video_id in enumerate(video_ids):
         video_links.append(video_id.get_attribute("href"))
         video_titles.append(video_id.get_attribute("title"))
-        
+
     video_infos = driver.find_elements(
         By.XPATH, "//span[@class='inline-metadata-item style-scope ytd-video-meta-block']")
     # for view_count in view_counts:
@@ -63,15 +64,17 @@
         "return document.documentElement.scrollHeight")
     driver.execute_script(
         f"window.scrollTo(0, {document_height_before + scroll_height});")
-    
+
     # write to file
     with open(f"results/{file_name}.txt", "a") as f:
         for i, video_link in enumerate(video_links):
             if i < len(video_links) - 1:
-                f.write(f"{video_link} ‽ {video_titles[i]} ‽ {video_views[i]} ‽ {video_published_times[i]}\n")
+                f.write(
+                    f"{video_link} ‽ {video_titles[i]} ‽ {video_views[i]} ‽ {video_published_times[i]}\n")
             else:
-                f.write(f"{video_link} ‽ {video_titles[i]} ‽ {video_views[i]} ‽ {video_published_times[i]}")
-        
+                f.write(
+                    f"{video_link} ‽ {video_titles[i]} ‽ {video_views[i]} ‽ {video_published_times[i]}")
+
     time.sleep(3)
     document_height_after = driver.execute_script(
         "return document.documentElement.scrollHeight")

results/gudang garam_scroll-10_20230122_231745.txt

@@ -0,0 +1,104 @@
+from selenium import webdriver
+import time
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import urllib.parse
+import datetime
+import os
+
+# options = webdriver.ChromeOptions()
+# # options.add_argument("start-maximized")
+# options.add_experimental_option("excludeSwitches", ["enable-automation"])
+# options.add_experimental_option('useAutomationExtension', False)
+
+options = webdriver.FirefoxOptions()
+options.add_argument("--headless")
+options.add_argument("--window-size=1920,1080")
+options.add_argument("--disable-gpu")
+options.add_argument("--disable-extensions")
+options.add_argument("--no-sandbox")
+options.add_argument("--disable-dev-shm-usage")
+options.add_argument("--disable-features=VizDisplayCompositor")
+options.add_argument("--disable-features=NetworkService")
+driver = webdriver.Firefox(options=options)
+BASE_DIR = os.path.join(os.path.dirname(__file__), '..')
+
+
+def handler(request, jsonify):
+    body = request.get_json()
+
+    if body is None:
+        return jsonify({'message': 'No body provided'}), 400
+
+    try:
+        query = body['query']
+        scroll = body['scroll']
+    except Exception as e:
+        return jsonify({'message': str(e) + " not provided"}), 400
+
+    query_url = urllib.parse.quote(query)
+    print('Query URL: ', query_url)
+    now = datetime.datetime.now()
+
+    driver.get(f"https://www.youtube.com/results?search_query={query_url}")
+
+    scroll_height = driver.execute_script("return window.innerHeight")
+    video_links = []
+    video_titles = []
+    video_views = []
+    video_published_times = []
+
+    max_scroll = scroll
+    file_name = f"{query}_scroll-{max_scroll}_{now.strftime('%Y%m%d_%H%M%S')}"
+
+    # while True:
+    while max_scroll > 0:
+        print("Scroll:", max_scroll)
+        max_scroll -= 1
+        video_ids = driver.find_elements(By.XPATH, "//a[@id='video-title']")
+        print('video_ids: ', video_ids)
+
+        for i, video_id in enumerate(video_ids):
+            print("videoTitle", video_id.get_attribute("title"))
+            print("videoID", video_id.get_attribute("href"))
+            video_links.append(video_id.get_attribute("href"))
+            video_titles.append(video_id.get_attribute("title"))
+
+        video_infos = driver.find_elements(
+            By.XPATH, "//span[@class='inline-metadata-item style-scope ytd-video-meta-block']")
+        # print('video_infos: ', video_infos)
+
+        for i, video_info in enumerate(video_infos):
+            if "views" in video_info.text:
+                view_count = video_info.text
+                video_views.append(view_count)
+            elif "ago" in video_info.text:
+                published_time = video_info.text
+                video_published_times.append(published_time)
+
+        document_height_before = driver.execute_script(
+            "return document.documentElement.scrollHeight")
+        driver.execute_script(
+            f"window.scrollTo(0, {document_height_before + scroll_height});")
+        
+        # write to file
+        with open(f"{BASE_DIR}/results/{file_name}.txt", "a") as f:
+            print(video_titles)
+            for i, video_link in enumerate(video_links):
+                print(video_link)
+                # if i < len(video_links) - 1:
+                #     f.write(
+                #         f"{video_link} ‽ {video_titles[i]} ‽ {video_views[i]} ‽ {video_published_times[i]}\n")
+                # else:
+                #     f.write(
+                #         f"{video_link} ‽ {video_titles[i]} ‽ {video_views[i]} ‽ {video_published_times[i]}")
+
+        time.sleep(1.5)
+        document_height_after = driver.execute_script(
+            "return document.documentElement.scrollHeight")
+        if document_height_after == document_height_before:
+            break
+
+    driver.quit()
+    return jsonify({'message': 'success', "filename": file_name}), 200