Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
193de54
Update site-list.py
iceybubble Sep 29, 2024
017c08a
fix: Add error messages to BabyRu to prevent false positives
Fandroid745 Oct 5, 2025
e3066a1
fix:added the username_claimed field
Fandroid745 Oct 5, 2025
4010a58
fix: changed the username_claimed to example placeholder
Fandroid745 Oct 5, 2025
01bca6b
fix: corrected the regexCheck field value to an empty string
Fandroid745 Oct 6, 2025
068fff8
fix:Remove regexCheck field and changed encoding to UTF-8
Fandroid745 Oct 7, 2025
53840c6
Merge branch 'master' into fix/babyru-false-positive
Fandroid745 Oct 7, 2025
3e4d9bc
Add TikTok support to Sherlock
VivekGaddam Oct 9, 2025
659bf92
corrected the errorMsg
VivekGaddam Oct 9, 2025
70b5055
corrected india F+ prevent
VivekGaddam Oct 11, 2025
99067b2
Add imood.com support
matheusfelipeog Oct 11, 2025
e774b08
Add imood.com support (#2647)
matheusfelipeog Oct 11, 2025
1645828
Add TikTok site support
aryanj10 Oct 11, 2025
d6256e9
classifiers for supported python version
Nirzak Oct 11, 2025
dc307fc
feat: add TikTok and Pinterest site detection support
aryanj10 Oct 11, 2025
0c457e5
Merge branch 'master' into fix/babyru-false-positive
Fandroid745 Oct 11, 2025
6d1280e
Merge pull request #2651 from aryanj10/add-tiktok-pinterest
ppfeister Oct 11, 2025
3edb73c
Merge pull request #2650 from Nirzak/patch-1
ppfeister Oct 11, 2025
d2072e2
chore: rem tiktok for improved rev
ppfeister Oct 11, 2025
28b78e7
Merge pull request #2633 from VivekGaddam/add-tiktok-support
ppfeister Oct 11, 2025
ef55f7d
chore: reformat json
ppfeister Oct 11, 2025
b9a72b5
fix: use Unicode escape sequences for BabyRu error messages
Fandroid745 Oct 11, 2025
79973a5
Update file handling to include encoding and correct comments
matheusfelipeog Oct 11, 2025
9483886
Cleanup site-list.py (#2307)
matheusfelipeog Oct 11, 2025
bd49aac
Merge pull request #2606 from Fandroid745/fix/babyru-false-positive
ppfeister Oct 11, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 17 additions & 11 deletions devel/site-list.py
Original file line number Diff line number Diff line change
@@ -1,39 +1,45 @@
#!/usr/bin/env python
# This module generates the listing of supported sites which can be found in
# sites.md. It also organizes all the sites in alphanumeric order
# sites.mdx. It also organizes all the sites in alphanumeric order
import json
import os


DATA_REL_URI: str = "sherlock_project/resources/data.json"

DEFAULT_ENCODING = "utf-8"

# Read the data.json file
with open(DATA_REL_URI, "r", encoding="utf-8") as data_file:
with open(DATA_REL_URI, "r", encoding=DEFAULT_ENCODING) as data_file:
data: dict = json.load(data_file)

# Removes schema-specific keywords for proper processing
social_networks: dict = dict(data)
social_networks = data.copy()
social_networks.pop('$schema', None)

# Sort the social networks in alphanumeric order
social_networks: list = sorted(social_networks.items())
social_networks = sorted(social_networks.items())

# Make output dir where the site list will be written
os.mkdir("output")

# Write the list of supported sites to sites.md
with open("output/sites.mdx", "w") as site_file:
site_file.write("---\ntitle: 'List of supported sites'\nsidebarTitle: 'Supported sites'\nicon: 'globe'\ndescription: 'Sherlock currently supports **400+** sites'\n---\n\n")
# Write the list of supported sites to sites.mdx
with open("output/sites.mdx", "w", encoding=DEFAULT_ENCODING) as site_file:
site_file.write("---\n")
site_file.write("title: 'List of supported sites'\n")
site_file.write("sidebarTitle: 'Supported sites'\n")
site_file.write("icon: 'globe'\n")
site_file.write("description: 'Sherlock currently supports **400+** sites'\n")
site_file.write("---\n\n")

for social_network, info in social_networks:
url_main = info["urlMain"]
is_nsfw = "**(NSFW)**" if info.get("isNSFW") else ""
site_file.write(f"1. [{social_network}]({url_main}) {is_nsfw}\n")

# Overwrite the data.json file with sorted data
with open(DATA_REL_URI, "w") as data_file:
with open(DATA_REL_URI, "w", encoding=DEFAULT_ENCODING) as data_file:
sorted_data = json.dumps(data, indent=2, sort_keys=True)
data_file.write(sorted_data)
data_file.write("\n")
data_file.write("\n") # Keep the newline after writing data

print("Finished updating supported site listing!")

4 changes: 4 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ classifiers = [
"Natural Language :: English",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Topic :: Security"
]
homepage = "https://sherlockproject.xyz/"
Expand Down
76 changes: 51 additions & 25 deletions sherlock_project/resources/data.json
Original file line number Diff line number Diff line change
Expand Up @@ -671,10 +671,7 @@
"url": "https://discord.com",
"urlMain": "https://discord.com/",
"urlProbe": "https://discord.com/api/v9/unique-username/username-attempt-unauthed",
"errorMsg": [
"{\"taken\":false}",
"The resource is being rate limited"
],
"errorMsg": ["{\"taken\":false}", "The resource is being rate limited"],
"request_method": "POST",
"request_payload": {
"username": "{}"
Expand Down Expand Up @@ -1049,10 +1046,7 @@
},
"HackerNews": {
"__comment__": "First errMsg invalid, second errMsg rate limited. Not ideal. Adjust for better rate limit filtering.",
"errorMsg": [
"No such user.",
"Sorry."
],
"errorMsg": ["No such user.", "Sorry."],
"errorType": "message",
"url": "https://news.ycombinator.com/user?id={}",
"urlMain": "https://news.ycombinator.com/",
Expand Down Expand Up @@ -1152,10 +1146,10 @@
"username_claimed": "blue"
},
"Ifunny": {
"errorType":"status_code",
"url":"https://ifunny.co/user/{}",
"urlMain":"https://ifunny.co/",
"username_claimed":"agua"
"errorType": "status_code",
"url": "https://ifunny.co/user/{}",
"urlMain": "https://ifunny.co/",
"username_claimed": "agua"
},
"IRC-Galleria": {
"errorType": "response_url",
Expand Down Expand Up @@ -1191,6 +1185,12 @@
"urlProbe": "https://api.imgur.com/account/v1/accounts/{}?client_id=546c25a59c58ad7",
"username_claimed": "blue"
},
"imood": {
"errorType": "status_code",
"url": "https://www.imood.com/users/{}",
"urlMain": "https://www.imood.com/",
"username_claimed": "blue"
},
"Instagram": {
"errorType": "status_code",
"url": "https://instagram.com/{}",
Expand Down Expand Up @@ -1492,14 +1492,13 @@
"urlMain": "https://www.motorradfrage.net/",
"username_claimed": "gutefrage"
},
"MuseScore": {
"errorType": "status_code",
"url": "https://musescore.com/{}",
"urlMain": "https://musescore.com/",
"username_claimed": "arrangeme",
"request_method": "GET"
},

"MuseScore": {
"errorType": "status_code",
"url": "https://musescore.com/{}",
"urlMain": "https://musescore.com/",
"username_claimed": "arrangeme",
"request_method": "GET"
},

"MyAnimeList": {
"errorType": "status_code",
Expand Down Expand Up @@ -2182,6 +2181,16 @@
"urlProbe": "https://ch.tetr.io/api/users/{}",
"username_claimed": "osk"
},
"TikTok": {
"url": "https://www.tiktok.com/@{}",
"urlMain": "https://www.tiktok.com",
"errorType": "message",
"errorMsg": [
"\"statusCode\":10221",
"Govt. of India decided to block 59 apps"
],
"username_claimed": "charlidamelio"
},
"Tiendanube": {
"url": "https://{}.mitiendanube.com/",
"urlMain": "https://www.tiendanube.com/",
Expand Down Expand Up @@ -2374,9 +2383,7 @@
"username_claimed": "red"
},
"Venmo": {
"errorMsg": [
"Venmo | Page Not Found"
],
"errorMsg": ["Venmo | Page Not Found"],
"errorType": "message",
"headers": {
"Host": "account.venmo.com"
Expand Down Expand Up @@ -2879,6 +2886,14 @@
"urlMain": "https://pikabu.ru/",
"username_claimed": "blue"
},
"Pinterest": {
"errorType": "status_code",
"errorUrl": "https://www.pinterest.com/",
"url": "https://www.pinterest.com/{}/",
"urlProbe": "https://www.pinterest.com/oembed.json?url=https://www.pinterest.com/{}/",
"urlMain": "https://www.pinterest.com/",
"username_claimed": "blue"
},
"pr0gramm": {
"errorType": "status_code",
"url": "https://pr0gramm.com/user/{}",
Expand Down Expand Up @@ -2978,5 +2993,16 @@
"urlMain": "https://platzi.com/",
"username_claimed": "freddier",
"request_method": "GET"
}
}
},
"BabyRu": {
"url": "https://www.baby.ru/u/{}",
"urlMain": "https://www.baby.ru/",
"errorType": "message",
"errorMsg": [
"\u0421\u0442\u0440\u0430\u043d\u0438\u0446\u0430, \u043a\u043e\u0442\u043e\u0440\u0443\u044e \u0432\u044b \u0438\u0441\u043a\u0430\u043b\u0438, \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d\u0430",
"\u0414\u043e\u0441\u0442\u0443\u043f \u0441 \u0432\u0430\u0448\u0435\u0433\u043e IP-\u0430\u0434\u0440\u0435\u0441\u0430 \u0432\u0440\u0435\u043c\u0435\u043d\u043d\u043e \u043e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d"
],
"regexCheck": "",
"username_claimed": "example"
}
}
Loading