Skip to content

Commit 30002bd

Browse files
RootLUGRootLUG
committed
- migrate secrets leak from yara rules to semantic regexes
- add mirror json, pypi package list and ast pattern cache to mirror prefetching
1 parent 446a780 commit 30002bd

File tree

5 files changed

+194
-19
lines changed

5 files changed

+194
-19
lines changed

‎aura/analyzers/pypirc.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
import configparser
2+
3+
from .. import config
4+
from .detections import Detection
5+
from ..uri_handlers.base import ScanLocation
6+
from ..utils import Analyzer, fast_checksum
7+
from ..type_definitions import AnalyzerReturnType
8+
9+
10+
@Analyzer.ID("pypirc")
11+
def analyze(*, location: ScanLocation) -> AnalyzerReturnType:
12+
"""
13+
Scans for exposure of credentials inside the `.pypirc` file
14+
"""
15+
if location.location.name != ".pypirc":
16+
return
17+
18+
pypirc = configparser.ConfigParser()
19+
pypirc.read(str(location.location))
20+
21+
# Filter on these values, some pregenerated configurations use them and we don't want to generate false positives on these
22+
user_blacklist = config.CFG.get("pypirc", {}).get("username_blacklist", [])
23+
pwd_blacklist = config.CFG.get("pypirc", {}).get("password_blacklist", [])
24+
25+
for section_name in pypirc.sections():
26+
section = pypirc[section_name]
27+
28+
if "username" in section and "password" in section:
29+
username = section.get("username")
30+
password = section.get("password")
31+
32+
if username in user_blacklist or password in pwd_blacklist:
33+
continue
34+
35+
sig = fast_checksum(f"{section_name}#{username}#{password}")
36+
37+
yield Detection(
38+
detection_type="LeakingPyPIrc",
39+
message = "Leaking credentials in the `.pypirc` file",
40+
signature = f"pypirc#{sig}",
41+
location = location.location,
42+
score = 100, # TODO: make the score configurable
43+
extra = {
44+
"username": username,
45+
"password": password
46+
},
47+
tags = {"sensitive_file", "secrets_leak"}
48+
)
49+

‎aura/data/rules.yara‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -94,38 +94,23 @@ rule PossiblePersistence: persistence
9494
}
9595

9696

97-
rule SecretsLeak: secrets_leak
97+
/*rule SecretsLeak: secrets_leak
9898
{
9999
meta:
100100
score = 100
101101
strings:
102-
$aws_manager_id = /(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}/
103102
$aws_secret_key = /aws(.{0,20})?['\"][0-9a-zA-Z\/+]{40}['\"]/ nocase
104-
$aws_mws_key = /amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
105-
$facebook_secret_key = /(facebook|fb)(.{0,20})?['\"][0-9a-f]{32}['\"]/ nocase
106-
$facebook_client_id = /(facebook|fb)(.{0,20})?['\"][0-9]{13,17}['\"]/ nocase
107-
$twitter_secret_key = /twitter(.{0,20})?[0-9a-z]{35,44}/ nocase
108-
$twitter_client_id = /twitter(.{0,20})?[0-9a-z]{18,25}/ nocase
109103
$github = /github(.{0,20})?[0-9a-zA-Z]{35,40}/ nocase
110104
$linkedin_client_id = /linkedin(.{0,20})?[0-9a-z]{12}/ nocase
111105
$linkedin_secret_key = /linkedin(.{0,20})?[0-9a-z]{16}/ nocase
112106
$slack = /xox(b|a|p|r|s)-([0-9a-z]{10,48})?/ nocase
113-
$google_api_key = /AIza[0-9A-Za-z\\-_]{35}/
114107
$google_gcp_account = /"type": "service_account"/
115108
$heroku_api_key = /heroku(.{0,20})?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/ nocase
116-
$mailchimp_api_key = /(mailchimp|mc)(.{0,20})?[0-9a-f]{32}-us[0-9]{1,2}/ nocase
117-
$mailgun_api_key = /((mailgun|mg)(.{0,20})?)?key-[0-9a-z]{32}/ nocase
118-
$paypal_braintree_access_token = /access_token\$production\$[0-9a-z]{16}\$[0-9a-f]{32}/
119-
$picatic_api_key = /sk_live_[0-9a-z]{32}/
120109
$sendgrid_api_key = /SG\.[\w_]{16,32}\.[\w_]{16,64}/
121110
$slack_webhook = /https:\/\/hooks.slack.com\/services\/T[a-zA-Z0-9_]{8}\/B[a-zA-Z0-9_]{8}\/[a-zA-Z0-9_]{24}/
122-
$strip_api_key = /stripe(.{0,20})?[sr]k_live_[0-9a-zA-Z]{24}/ nocase
123-
$square_access_token = /sq0atp-[0-9A-Za-z\-_]{22}/
124-
$square_oauth_secret = /sq0csp-[0-9A-Za-z\\-_]{43}/
125-
$twilio_api_key = /twilio(.{0,20})?SK[0-9a-f]{32}/ nocase
126111
condition:
127112
any of them
128-
}
113+
}*/
129114

130115

131116
//TO-DO: https://github.com/SublimeCodeIntel/CodeIntel/blob/master/codeintel/which.py#L101

‎aura/data/signatures.yaml‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -622,3 +622,127 @@ strings: &default_strings
622622
pattern: "^sudo .*$"
623623
message: "Executing sudo command"
624624
score: 20
625+
626+
# Regexes for detecting leaking api tokens, secrets, etc...
627+
# Source of some regexes used: https://www.ndss-symposium.org/wp-content/uploads/2019/02/ndss2019_04B-3_Meli_paper.pdf
628+
- id: twitter_access_token
629+
type: regex
630+
pattern: "^[1-9][0-9]+-[0-9a-zA-Z]{40}$"
631+
message: "Twitter access token"
632+
score: 100
633+
tags:
634+
- secrets_leak
635+
636+
- id: facebook_access_token
637+
type: regex
638+
pattern: "^EAACEdEose0cBA[0-9A-Za-z]+$"
639+
message: "Facebook access token"
640+
score: 100
641+
tags:
642+
- secrets_leak
643+
644+
- id: google_api_key
645+
type: regex
646+
pattern: "^AIza[-0-9A-Za-z_]{35}$"
647+
message: "Google API key"
648+
score: 100
649+
tags:
650+
- secrets_leak
651+
652+
- id: google_oauth_id
653+
type: regex
654+
pattern: "^[0-9]+-[0-9A-Za-z_]{32}\\.apps\\.googleusercontent\\.com$"
655+
message: "Google OAuth ID"
656+
score: 100
657+
tags:
658+
- secrets_leak
659+
660+
- id: picatic_api_key
661+
type: regex
662+
pattern: "^sk_live_[0-9a-z]{32}$"
663+
message: "Picatic API key"
664+
score: 100
665+
tags:
666+
- secrets_leak
667+
668+
- id: stripe_standard_key
669+
type: regex
670+
pattern: "^sk_live_[0-9a-zA-Z]{24}$"
671+
message: "Stripe standard key"
672+
score: 100
673+
tags:
674+
- secrets_leak
675+
676+
- id: stripe_restricted_key
677+
type: regex
678+
pattern: "^rk_live_[0-9a-zA-Z]{24}$"
679+
message: "Stripe restricted key"
680+
score: 100
681+
tags:
682+
- secrets_leak
683+
684+
- id: square_access_token
685+
type: regex
686+
pattern: "^sq0atp-[-0-9A-Za-z_]{22}$"
687+
message: "Square access token"
688+
score: 100
689+
tags:
690+
- secrets_leak
691+
692+
- id: square_oauth_secret
693+
type: regex
694+
pattern: "^sq0csp-[-0-9A-Za-z_]{43}$"
695+
message: "Square OAuth secret"
696+
score: 100
697+
tags:
698+
- secrets_leak
699+
700+
- id: paypal_braintree
701+
type: regex
702+
pattern: "^access_token\\$production\\$[0-9a-z]{16}\\$[0-9a-f]{32}$"
703+
message: "PayPal braintree access token"
704+
score: 100
705+
tags:
706+
- secrets_leak
707+
708+
- id: amazon_mws_auth_token
709+
type: regex
710+
pattern: "^amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
711+
message: "Amazon MWS auth token"
712+
score: 100
713+
tags:
714+
- secrets_leak
715+
716+
- id: twilio_api_key
717+
type: regex
718+
pattern: "^SK[0-9a-fA-F]{32}$"
719+
message: "Twilio API key"
720+
score: 100
721+
tags:
722+
- secrets_leak
723+
724+
- id: mailgun_api_key
725+
type: regex
726+
pattern: "^key-[0-9a-zA-Z]{32}$"
727+
message: "Mailgun API key"
728+
score: 100
729+
tags:
730+
- secrets_leak
731+
732+
- id: mailchimp_api_key
733+
type: regex
734+
pattern: "^[0-9a-f]{32}-us[0-9]{1,2}$"
735+
message: "MailChimp API key"
736+
score: 100
737+
tags:
738+
- secrets_leak
739+
740+
- id: amazon_aws_key
741+
type: regex
742+
pattern: "^(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[0-9A-Z]{16}$"
743+
message: "Amazon AWS key"
744+
score: 100
745+
tags:
746+
- secrets_leak
747+
748+

‎aura/mirror.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,9 @@ def get_mirror_path(cls) -> typing.Optional[Path]:
2525

2626
return cls._mirror_path
2727

28-
def list_packages(self) -> typing.Generator[Path, None, None]:
29-
yield from (self.get_mirror_path() / "json").iterdir()
28+
@classmethod
29+
def list_packages(cls) -> typing.Generator[Path, None, None]:
30+
yield from (cls.get_mirror_path() / "json").iterdir()
3031

3132
def get_json(self, package_name) -> dict:
3233
assert package_name

‎aura/prefetch.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,11 @@
22
import logging
33
from typing import Iterable, TextIO
44

5+
import tqdm
6+
57
from . import github
8+
from . import mirror
9+
from . import cache
610
from .worker_executor import non_blocking, AsyncQueue
711
from .exceptions import NoSuchPackage
812
from .uri_handlers.base import URIHandler
@@ -34,6 +38,18 @@ async def fetch_package(uri_queue, github_prefetcher):
3438

3539

3640
def prefetch_mirror(uris: Iterable[str], workers=10):
41+
logger.info("Caching AST patterns")
42+
cache.ASTPatternCache.proxy()
43+
44+
logger.info("Caching list of packages on pypi")
45+
cache.PyPIPackageList.proxy()
46+
47+
logger.info("Prefetching package JSON information")
48+
lm = mirror.LocalMirror()
49+
pkgs = tuple(lm.list_packages())
50+
for x in tqdm.tqdm(pkgs, leave=False):
51+
lm.get_json(x)
52+
3753
loop = asyncio.get_event_loop()
3854

3955
pf = github.GitHubPrefetcher()

0 commit comments

Comments
 (0)