|
1 | 1 | #!/usr/bin/env python3 |
2 | 2 |
|
| 3 | +# fmt: off |
3 | 4 | # import argparse |
| 5 | +import arrow |
4 | 6 | # import git |
5 | 7 | # import numpy as np |
6 | 8 | # import os |
7 | | -# import pandas as pd |
| 9 | +import pandas as pd |
8 | 10 | # import re |
9 | 11 | # import requests |
10 | 12 | # import requests_cache |
| 13 | +# import subprocess |
| 14 | +# import time |
11 | 15 | # from bs4 import BeautifulSoup, Comment |
12 | | -# from datetime import timedelta |
13 | | -# from decouple import config |
| 16 | +from datetime import timedelta |
| 17 | +from decouple import config |
| 18 | +# from functools import wraps |
14 | 19 | # from icecream import ic |
15 | 20 | # from <local.py_module> import * |
16 | 21 | from pathlib import Path |
17 | 22 | # from playwright.async_api import async_playwright |
18 | 23 | # from playwright.sync_api import sync_playwright |
19 | 24 | # from prettytable import PrettyTable |
20 | 25 | # from requests_cache import CachedSession |
| 26 | +# fmt: on |
21 | 27 |
|
22 | 28 | """ |
23 | 29 | The commented out section is boilerplate for common operations. |
24 | 30 | Feel free to uncomment and/or delete after first commit. |
25 | 31 | """ |
26 | | -## env |
| 32 | +# env |
27 | 33 | home = Path.home() |
28 | | -# now = datetime.datetime.now() |
29 | | -# out = f"{home}/Downloads/result_{now:%Y%m%d_%H%M%S}.csv" |
30 | | -env = Path('.env') |
| 34 | +cwd = Path.cwd() |
| 35 | +now = arrow.now().format('YYYYMMDD_HHmmss') |
| 36 | +out = f"{cwd}/formatted/results_{now}.csv" |
| 37 | + |
| 38 | +# env vars (hierachy: args, env, .env) |
| 39 | +HOST = config('HOST', default='localhost') |
| 40 | +USER = config('USER') |
| 41 | +PASS = config('PASS') |
31 | 42 |
|
32 | 43 | ## verbose icecream |
33 | 44 | # ic.configureOutput(includeContext=True) |
34 | 45 |
|
35 | | -## pwd |
36 | | -cwd = Path.cwd() |
37 | | -# print(f"Current working directory: {cwd}") |
38 | 46 |
|
39 | | -## create file and parent directories |
40 | | -# meta_file = f"{cwd}/metadata/metadata.json" |
41 | | -# Path(meta_file).parents[0].mkdir(parents=True, exist_ok=True) |
| 47 | +# def timeit(func): |
| 48 | +# @wraps(func) |
| 49 | +# def timeit_wrapper(*args, **kwargs): |
| 50 | +# start_time = time.perf_counter() |
| 51 | +# result = func(*args, **kwargs) |
| 52 | +# end_time = time.perf_counter() |
| 53 | +# total_time = end_time - start_time |
| 54 | +# print(f"Function {func.__name__}{args} {kwargs} Took {total_time:.4f} seconds") |
| 55 | +# return result |
| 56 | + |
| 57 | +# return timeit_wrapper |
| 58 | + |
| 59 | + |
| 60 | +# @timeit |
| 61 | +# def sub_get_url(id): |
| 62 | +# cmd = f"mas info {id} | awk '/https/ {{print $NF}}'" |
| 63 | + |
| 64 | +# return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) |
| 65 | + |
| 66 | + |
| 67 | +# # 0.1730, 1.7148 seconds |
| 68 | +# sub_get_url(490179405) |
| 69 | + |
42 | 70 |
|
43 | 71 | # # clone substrapunks repo |
44 | 72 | # if not Path(cwd/'substrapunks').exists(): |
|
64 | 92 | # print("Exiting...") |
65 | 93 | # exit() |
66 | 94 |
|
67 | | -# if cwd != dir_path: |
68 | | -# os.chdir(dir_path) |
69 | | -# print(os.getcwd()) |
70 | | - |
71 | 95 | # folders = ['logs', 'user_data'] |
72 | 96 |
|
73 | 97 | # for folder in folders: |
74 | 98 | # if not Path(folder).exists(): |
75 | 99 | # os.mkdir(os.path.join(cwd, folder)) |
76 | 100 |
|
77 | | -# # creds |
78 | | -# if env.exists(): |
79 | | -# HOST = config('HOST', default='localhost') |
80 | | -# USER = config('USER') |
81 | | -# PASS = config('PASS') |
82 | | -# else: |
83 | | -# HOST = os.getenv('HOST', default='localhost') |
84 | | -# USER = os.getenv('USER') |
85 | | -# PASS = os.getenv('PASS') |
86 | | - |
87 | 101 | ## mkdir -p ./csv && cd $_ |
88 | 102 | # if Path('csv').exists(): |
89 | 103 | # os.chdir('./csv') |
|
0 commit comments