diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 3b768a49..f58ebd80 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -10,7 +10,7 @@ on: jobs: lint: name: Lint Test - if: "${{ !endsWith(github.actor, '[bot]') }}" + if: ${{ !endsWith(github.actor, '[bot]') }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index c7017878..98dce07a 100644 --- a/README.md +++ b/README.md @@ -92,9 +92,10 @@ Usage: Flags: INPUT: - -l, -list string input file containing list of hosts to process - -rr, -request string file containing raw request - -u, -target string[] input target host(s) to probe + -l, -list string input file containing list of hosts to process + -rr, -request string file containing raw request + -u, -target string[] input target host(s) to probe + -im, -input-mode string mode of input file (burp) PROBES: -sc, -status-code display response status-code @@ -110,9 +111,11 @@ PROBES: -title display page title -bp, -body-preview display first N characters of response body (default 100) -server, -web-server display server name - -td, -tech-detect display technology in use based on wappalyzer dataset + -td, -tech-detect display technology in use based on wappalyzer dataset -cff, -custom-fingerprint-file string path to a custom fingerprint file for technology detection - -method display http request method + -cpe display CPE (Common Platform Enumeration) based on awesome-search-queries + -wp, -wordpress display WordPress plugins and themes + -method display http request method -ws, -websocket display server using websocket -ip display host ip -cname display host cname @@ -231,6 +234,7 @@ CONFIGURATIONS: -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization -no-stdin Disable Stdin processing -hae, -http-api-endpoint string experimental http api endpoint + -sf, -secret-file string path to secret file for authentication DEBUG: -health-check, -hc run diagnostic check up @@ -277,9 +281,28 @@ For details about running httpx, see https://docs.projectdiscovery.io/tools/http # Notes - As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable. +- Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp` - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) +- Secret files can be used for domain-based authentication via `-sf secrets.yaml`. Supported auth types: `BasicAuth`, `BearerToken`, `Header`, `Cookie`, `Query`. Example: + ```yaml + id: example-auth + info: + name: Example Auth Config + static: + - type: Header + domains: + - api.example.com + headers: + - key: X-API-Key + value: secret-key-here + - type: BasicAuth + domains-regex: + - ".*\\.internal\\.com$" + username: admin + password: secret + ``` - The following flags should be used for specific use cases instead of running them as default with other probes: - `-ports` - `-path` @@ -307,4 +330,4 @@ Probing feature is inspired by [@tomnomnom/httprobe](https://github.com/tomnomno Join Discord - + \ No newline at end of file diff --git a/common/authprovider/authx/basic_auth.go b/common/authprovider/authx/basic_auth.go new file mode 100644 index 00000000..b7579066 --- /dev/null +++ b/common/authprovider/authx/basic_auth.go @@ -0,0 +1,31 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" +) + +var ( + _ AuthStrategy = &BasicAuthStrategy{} +) + +// BasicAuthStrategy is a strategy for basic auth +type BasicAuthStrategy struct { + Data *Secret +} + +// NewBasicAuthStrategy creates a new basic auth strategy +func NewBasicAuthStrategy(data *Secret) *BasicAuthStrategy { + return &BasicAuthStrategy{Data: data} +} + +// Apply applies the basic auth strategy to the request +func (s *BasicAuthStrategy) Apply(req *http.Request) { + req.SetBasicAuth(s.Data.Username, s.Data.Password) +} + +// ApplyOnRR applies the basic auth strategy to the retryable request +func (s *BasicAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + req.SetBasicAuth(s.Data.Username, s.Data.Password) +} diff --git a/common/authprovider/authx/bearer_auth.go b/common/authprovider/authx/bearer_auth.go new file mode 100644 index 00000000..edf6f439 --- /dev/null +++ b/common/authprovider/authx/bearer_auth.go @@ -0,0 +1,31 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" +) + +var ( + _ AuthStrategy = &BearerTokenAuthStrategy{} +) + +// BearerTokenAuthStrategy is a strategy for bearer token auth +type BearerTokenAuthStrategy struct { + Data *Secret +} + +// NewBearerTokenAuthStrategy creates a new bearer token auth strategy +func NewBearerTokenAuthStrategy(data *Secret) *BearerTokenAuthStrategy { + return &BearerTokenAuthStrategy{Data: data} +} + +// Apply applies the bearer token auth strategy to the request +func (s *BearerTokenAuthStrategy) Apply(req *http.Request) { + req.Header.Set("Authorization", "Bearer "+s.Data.Token) +} + +// ApplyOnRR applies the bearer token auth strategy to the retryable request +func (s *BearerTokenAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + req.Header.Set("Authorization", "Bearer "+s.Data.Token) +} diff --git a/common/authprovider/authx/cookies_auth.go b/common/authprovider/authx/cookies_auth.go new file mode 100644 index 00000000..0d7872b2 --- /dev/null +++ b/common/authprovider/authx/cookies_auth.go @@ -0,0 +1,62 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" +) + +var ( + _ AuthStrategy = &CookiesAuthStrategy{} +) + +// CookiesAuthStrategy is a strategy for cookies auth +type CookiesAuthStrategy struct { + Data *Secret +} + +// NewCookiesAuthStrategy creates a new cookies auth strategy +func NewCookiesAuthStrategy(data *Secret) *CookiesAuthStrategy { + return &CookiesAuthStrategy{Data: data} +} + +// Apply applies the cookies auth strategy to the request +func (s *CookiesAuthStrategy) Apply(req *http.Request) { + for _, cookie := range s.Data.Cookies { + req.AddCookie(&http.Cookie{ + Name: cookie.Key, + Value: cookie.Value, + }) + } +} + +// ApplyOnRR applies the cookies auth strategy to the retryable request +func (s *CookiesAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + // Build a set of cookie names to replace + newCookieNames := make(map[string]struct{}, len(s.Data.Cookies)) + for _, cookie := range s.Data.Cookies { + newCookieNames[cookie.Key] = struct{}{} + } + + // Filter existing cookies, keeping only those not being replaced + existingCookies := req.Cookies() + filteredCookies := make([]*http.Cookie, 0, len(existingCookies)) + for _, cookie := range existingCookies { + if _, shouldReplace := newCookieNames[cookie.Name]; !shouldReplace { + filteredCookies = append(filteredCookies, cookie) + } + } + + // Clear and reset cookies + req.Header.Del("Cookie") + for _, cookie := range filteredCookies { + req.AddCookie(cookie) + } + // Add new cookies + for _, cookie := range s.Data.Cookies { + req.AddCookie(&http.Cookie{ + Name: cookie.Key, + Value: cookie.Value, + }) + } +} diff --git a/common/authprovider/authx/file.go b/common/authprovider/authx/file.go new file mode 100644 index 00000000..ca35568b --- /dev/null +++ b/common/authprovider/authx/file.go @@ -0,0 +1,244 @@ +package authx + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/projectdiscovery/utils/errkit" + "github.com/projectdiscovery/utils/generic" + stringsutil "github.com/projectdiscovery/utils/strings" + "gopkg.in/yaml.v3" +) + +type AuthType string + +const ( + BasicAuth AuthType = "BasicAuth" + BearerTokenAuth AuthType = "BearerToken" + HeadersAuth AuthType = "Header" + CookiesAuth AuthType = "Cookie" + QueryAuth AuthType = "Query" +) + +// SupportedAuthTypes returns the supported auth types +func SupportedAuthTypes() []string { + return []string{ + string(BasicAuth), + string(BearerTokenAuth), + string(HeadersAuth), + string(CookiesAuth), + string(QueryAuth), + } +} + +// Authx is a struct for secrets or credentials file +type Authx struct { + ID string `json:"id" yaml:"id"` + Info AuthFileInfo `json:"info" yaml:"info"` + Secrets []Secret `json:"static" yaml:"static"` +} + +type AuthFileInfo struct { + Name string `json:"name" yaml:"name"` + Author string `json:"author" yaml:"author"` + Severity string `json:"severity" yaml:"severity"` + Description string `json:"description" yaml:"description"` +} + +// Secret is a struct for secret or credential +type Secret struct { + Type string `json:"type" yaml:"type"` + Domains []string `json:"domains" yaml:"domains"` + DomainsRegex []string `json:"domains-regex" yaml:"domains-regex"` + Headers []KV `json:"headers" yaml:"headers"` // Headers preserve exact casing (useful for case-sensitive APIs) + Cookies []Cookie `json:"cookies" yaml:"cookies"` + Params []KV `json:"params" yaml:"params"` + Username string `json:"username" yaml:"username"` // can be either email or username + Password string `json:"password" yaml:"password"` + Token string `json:"token" yaml:"token"` // Bearer Auth token +} + +// GetStrategy returns the auth strategy for the secret +func (s *Secret) GetStrategy() AuthStrategy { + switch { + case strings.EqualFold(s.Type, string(BasicAuth)): + return NewBasicAuthStrategy(s) + case strings.EqualFold(s.Type, string(BearerTokenAuth)): + return NewBearerTokenAuthStrategy(s) + case strings.EqualFold(s.Type, string(HeadersAuth)): + return NewHeadersAuthStrategy(s) + case strings.EqualFold(s.Type, string(CookiesAuth)): + return NewCookiesAuthStrategy(s) + case strings.EqualFold(s.Type, string(QueryAuth)): + return NewQueryAuthStrategy(s) + } + return nil +} + +func (s *Secret) Validate() error { + if !stringsutil.EqualFoldAny(s.Type, SupportedAuthTypes()...) { + return fmt.Errorf("invalid type: %s", s.Type) + } + if len(s.Domains) == 0 && len(s.DomainsRegex) == 0 { + return fmt.Errorf("domains or domains-regex cannot be empty") + } + if len(s.DomainsRegex) > 0 { + for _, domain := range s.DomainsRegex { + _, err := regexp.Compile(domain) + if err != nil { + return fmt.Errorf("invalid domain regex: %s", domain) + } + } + } + + switch { + case strings.EqualFold(s.Type, string(BasicAuth)): + if s.Username == "" { + return fmt.Errorf("username cannot be empty in basic auth") + } + if s.Password == "" { + return fmt.Errorf("password cannot be empty in basic auth") + } + case strings.EqualFold(s.Type, string(BearerTokenAuth)): + if s.Token == "" { + return fmt.Errorf("token cannot be empty in bearer token auth") + } + case strings.EqualFold(s.Type, string(HeadersAuth)): + if len(s.Headers) == 0 { + return fmt.Errorf("headers cannot be empty in headers auth") + } + for _, header := range s.Headers { + if err := header.Validate(); err != nil { + return fmt.Errorf("invalid header in headersAuth: %s", err) + } + } + case strings.EqualFold(s.Type, string(CookiesAuth)): + if len(s.Cookies) == 0 { + return fmt.Errorf("cookies cannot be empty in cookies auth") + } + for _, cookie := range s.Cookies { + if cookie.Raw != "" { + if err := cookie.Parse(); err != nil { + return fmt.Errorf("invalid raw cookie in cookiesAuth: %s", err) + } + } + if err := cookie.Validate(); err != nil { + return fmt.Errorf("invalid cookie in cookiesAuth: %s", err) + } + } + case strings.EqualFold(s.Type, string(QueryAuth)): + if len(s.Params) == 0 { + return fmt.Errorf("query cannot be empty in query auth") + } + for _, query := range s.Params { + if err := query.Validate(); err != nil { + return fmt.Errorf("invalid query in queryAuth: %s", err) + } + } + } + return nil +} + +type KV struct { + Key string `json:"key" yaml:"key"` // Header key (preserves exact casing) + Value string `json:"value" yaml:"value"` +} + +func (k *KV) Validate() error { + if k.Key == "" { + return fmt.Errorf("key cannot be empty") + } + if k.Value == "" { + return fmt.Errorf("value cannot be empty") + } + return nil +} + +type Cookie struct { + Key string `json:"key" yaml:"key"` + Value string `json:"value" yaml:"value"` + Raw string `json:"raw" yaml:"raw"` +} + +func (c *Cookie) Validate() error { + if c.Raw != "" { + return nil + } + if c.Key == "" { + return fmt.Errorf("key cannot be empty") + } + if c.Value == "" { + return fmt.Errorf("value cannot be empty") + } + return nil +} + +// Parse parses the cookie +// in raw the cookie is in format of +// Set-Cookie: =; Expires=; Path=; Domain=; Secure; HttpOnly +func (c *Cookie) Parse() error { + if c.Raw == "" { + return fmt.Errorf("raw cookie cannot be empty") + } + tmp := strings.TrimPrefix(c.Raw, "Set-Cookie: ") + slice := strings.Split(tmp, ";") + if len(slice) == 0 { + return fmt.Errorf("invalid raw cookie no ; found") + } + // first element is the cookie name and value + // Use SplitN to preserve '=' characters in the cookie value + cookie := strings.SplitN(slice[0], "=", 2) + if len(cookie) != 2 { + return fmt.Errorf("invalid raw cookie: missing '=' in cookie name=value: %s", c.Raw) + } + c.Key = strings.TrimSpace(cookie[0]) + c.Value = strings.TrimSpace(cookie[1]) + if c.Key == "" { + return fmt.Errorf("invalid raw cookie: empty cookie name: %s", c.Raw) + } + return nil +} + +// GetAuthDataFromFile reads the auth data from file +func GetAuthDataFromFile(file string) (*Authx, error) { + ext := strings.ToLower(filepath.Ext(file)) + if !generic.EqualsAny(ext, ".yml", ".yaml", ".json") { + return nil, fmt.Errorf("invalid file extension: supported extensions are .yml,.yaml and .json got %s", ext) + } + bin, err := os.ReadFile(file) + if err != nil { + return nil, err + } + if ext == ".yml" || ext == ".yaml" { + return GetAuthDataFromYAML(bin) + } + return GetAuthDataFromJSON(bin) +} + +// GetAuthDataFromYAML reads the auth data from yaml +func GetAuthDataFromYAML(data []byte) (*Authx, error) { + var auth Authx + err := yaml.Unmarshal(data, &auth) + if err != nil { + errorErr := errkit.FromError(err) + errorErr.Msgf("could not unmarshal yaml") + return nil, errorErr + } + return &auth, nil +} + +// GetAuthDataFromJSON reads the auth data from json +func GetAuthDataFromJSON(data []byte) (*Authx, error) { + var auth Authx + err := json.Unmarshal(data, &auth) + if err != nil { + errorErr := errkit.FromError(err) + errorErr.Msgf("could not unmarshal json") + return nil, errorErr + } + return &auth, nil +} diff --git a/common/authprovider/authx/file_test.go b/common/authprovider/authx/file_test.go new file mode 100644 index 00000000..b42fc59b --- /dev/null +++ b/common/authprovider/authx/file_test.go @@ -0,0 +1,408 @@ +package authx + +import ( + "os" + "path/filepath" + "testing" +) + +func TestCookieParse(t *testing.T) { + tests := []struct { + name string + raw string + wantKey string + wantValue string + wantErr bool + }{ + { + name: "simple cookie", + raw: "session=abc123", + wantKey: "session", + wantValue: "abc123", + wantErr: false, + }, + { + name: "cookie with Set-Cookie prefix", + raw: "Set-Cookie: session=abc123; Path=/", + wantKey: "session", + wantValue: "abc123", + wantErr: false, + }, + { + name: "cookie with equals in value", + raw: "token=eyJhbGciOiJIUzI1NiJ9==; Path=/", + wantKey: "token", + wantValue: "eyJhbGciOiJIUzI1NiJ9==", + wantErr: false, + }, + { + name: "cookie with spaces", + raw: " session = abc123 ; Path=/", + wantKey: "session", + wantValue: "abc123", + wantErr: false, + }, + { + name: "empty raw", + raw: "", + wantErr: true, + }, + { + name: "missing equals", + raw: "sessionabc123; Path=/", + wantErr: true, + }, + { + name: "empty key", + raw: "=abc123; Path=/", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Cookie{Raw: tt.raw} + err := c.Parse() + + if (err != nil) != tt.wantErr { + t.Errorf("Cookie.Parse() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !tt.wantErr { + if c.Key != tt.wantKey { + t.Errorf("Cookie.Parse() Key = %v, want %v", c.Key, tt.wantKey) + } + if c.Value != tt.wantValue { + t.Errorf("Cookie.Parse() Value = %v, want %v", c.Value, tt.wantValue) + } + } + }) + } +} + +func TestSecretValidate(t *testing.T) { + tests := []struct { + name string + secret Secret + wantErr bool + }{ + { + name: "valid basic auth", + secret: Secret{ + Type: "BasicAuth", + Domains: []string{"example.com"}, + Username: "user", + Password: "pass", + }, + wantErr: false, + }, + { + name: "valid bearer token", + secret: Secret{ + Type: "BearerToken", + Domains: []string{"example.com"}, + Token: "abc123", + }, + wantErr: false, + }, + { + name: "valid header auth", + secret: Secret{ + Type: "Header", + Domains: []string{"example.com"}, + Headers: []KV{{Key: "X-API-Key", Value: "secret"}}, + }, + wantErr: false, + }, + { + name: "valid cookie auth", + secret: Secret{ + Type: "Cookie", + Domains: []string{"example.com"}, + Cookies: []Cookie{{Key: "session", Value: "abc123"}}, + }, + wantErr: false, + }, + { + name: "valid query auth", + secret: Secret{ + Type: "Query", + Domains: []string{"example.com"}, + Params: []KV{{Key: "api_key", Value: "secret"}}, + }, + wantErr: false, + }, + { + name: "invalid type", + secret: Secret{ + Type: "InvalidType", + Domains: []string{"example.com"}, + }, + wantErr: true, + }, + { + name: "missing domains", + secret: Secret{ + Type: "BasicAuth", + Username: "user", + Password: "pass", + }, + wantErr: true, + }, + { + name: "basic auth missing username", + secret: Secret{ + Type: "BasicAuth", + Domains: []string{"example.com"}, + Password: "pass", + }, + wantErr: true, + }, + { + name: "basic auth missing password", + secret: Secret{ + Type: "BasicAuth", + Domains: []string{"example.com"}, + Username: "user", + }, + wantErr: true, + }, + { + name: "bearer auth missing token", + secret: Secret{ + Type: "BearerToken", + Domains: []string{"example.com"}, + }, + wantErr: true, + }, + { + name: "header auth missing headers", + secret: Secret{ + Type: "Header", + Domains: []string{"example.com"}, + }, + wantErr: true, + }, + { + name: "cookie auth missing cookies", + secret: Secret{ + Type: "Cookie", + Domains: []string{"example.com"}, + }, + wantErr: true, + }, + { + name: "query auth missing params", + secret: Secret{ + Type: "Query", + Domains: []string{"example.com"}, + }, + wantErr: true, + }, + { + name: "valid domain regex", + secret: Secret{ + Type: "BearerToken", + DomainsRegex: []string{".*\\.example\\.com"}, + Token: "abc123", + }, + wantErr: false, + }, + { + name: "invalid domain regex", + secret: Secret{ + Type: "BearerToken", + DomainsRegex: []string{"[invalid"}, + Token: "abc123", + }, + wantErr: true, + }, + { + name: "case insensitive type", + secret: Secret{ + Type: "basicauth", + Domains: []string{"example.com"}, + Username: "user", + Password: "pass", + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.secret.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Secret.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestGetAuthDataFromFile(t *testing.T) { + // Create temp directory for test files + tmpDir := t.TempDir() + + yamlContent := `id: test +info: + name: test +static: + - type: BasicAuth + domains: + - example.com + username: user + password: pass +` + + jsonContent := `{ + "id": "test", + "info": {"name": "test"}, + "static": [ + { + "type": "BasicAuth", + "domains": ["example.com"], + "username": "user", + "password": "pass" + } + ] +}` + + tests := []struct { + name string + filename string + content string + wantErr bool + }{ + { + name: "yaml file lowercase", + filename: "secrets.yaml", + content: yamlContent, + wantErr: false, + }, + { + name: "yml file lowercase", + filename: "secrets.yml", + content: yamlContent, + wantErr: false, + }, + { + name: "json file lowercase", + filename: "secrets.json", + content: jsonContent, + wantErr: false, + }, + { + name: "yaml file uppercase", + filename: "secrets.YAML", + content: yamlContent, + wantErr: false, + }, + { + name: "json file uppercase", + filename: "secrets.JSON", + content: jsonContent, + wantErr: false, + }, + { + name: "invalid extension", + filename: "secrets.txt", + content: yamlContent, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create test file + filePath := filepath.Join(tmpDir, tt.filename) + err := os.WriteFile(filePath, []byte(tt.content), 0644) + if err != nil { + t.Fatalf("Failed to create test file: %v", err) + } + + _, err = GetAuthDataFromFile(filePath) + if (err != nil) != tt.wantErr { + t.Errorf("GetAuthDataFromFile() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestSecretGetStrategy(t *testing.T) { + tests := []struct { + name string + secret Secret + wantType string + wantNil bool + }{ + { + name: "basic auth strategy", + secret: Secret{ + Type: "BasicAuth", + Username: "user", + Password: "pass", + }, + wantType: "*authx.BasicAuthStrategy", + wantNil: false, + }, + { + name: "bearer token strategy", + secret: Secret{ + Type: "BearerToken", + Token: "abc123", + }, + wantType: "*authx.BearerTokenAuthStrategy", + wantNil: false, + }, + { + name: "header strategy", + secret: Secret{ + Type: "Header", + Headers: []KV{{Key: "X-API-Key", Value: "secret"}}, + }, + wantType: "*authx.HeadersAuthStrategy", + wantNil: false, + }, + { + name: "cookie strategy", + secret: Secret{ + Type: "Cookie", + Cookies: []Cookie{{Key: "session", Value: "abc123"}}, + }, + wantType: "*authx.CookiesAuthStrategy", + wantNil: false, + }, + { + name: "query strategy", + secret: Secret{ + Type: "Query", + Params: []KV{{Key: "api_key", Value: "secret"}}, + }, + wantType: "*authx.QueryAuthStrategy", + wantNil: false, + }, + { + name: "unknown type returns nil", + secret: Secret{ + Type: "UnknownType", + }, + wantNil: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + strategy := tt.secret.GetStrategy() + if tt.wantNil { + if strategy != nil { + t.Errorf("GetStrategy() = %T, want nil", strategy) + } + } else { + if strategy == nil { + t.Errorf("GetStrategy() = nil, want %s", tt.wantType) + } + } + }) + } +} diff --git a/common/authprovider/authx/headers_auth.go b/common/authprovider/authx/headers_auth.go new file mode 100644 index 00000000..d474f75b --- /dev/null +++ b/common/authprovider/authx/headers_auth.go @@ -0,0 +1,39 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" +) + +var ( + _ AuthStrategy = &HeadersAuthStrategy{} +) + +// HeadersAuthStrategy is a strategy for headers auth +type HeadersAuthStrategy struct { + Data *Secret +} + +// NewHeadersAuthStrategy creates a new headers auth strategy +func NewHeadersAuthStrategy(data *Secret) *HeadersAuthStrategy { + return &HeadersAuthStrategy{Data: data} +} + +// Apply applies the headers auth strategy to the request +// NOTE: This preserves exact header casing (e.g., barAuthToken stays as barAuthToken) +// This is useful for APIs that require case-sensitive header names +func (s *HeadersAuthStrategy) Apply(req *http.Request) { + for _, header := range s.Data.Headers { + req.Header[header.Key] = []string{header.Value} + } +} + +// ApplyOnRR applies the headers auth strategy to the retryable request +// NOTE: This preserves exact header casing (e.g., barAuthToken stays as barAuthToken) +// This is useful for APIs that require case-sensitive header names +func (s *HeadersAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + for _, header := range s.Data.Headers { + req.Header[header.Key] = []string{header.Value} + } +} diff --git a/common/authprovider/authx/query_auth.go b/common/authprovider/authx/query_auth.go new file mode 100644 index 00000000..796d8b1f --- /dev/null +++ b/common/authprovider/authx/query_auth.go @@ -0,0 +1,42 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" + urlutil "github.com/projectdiscovery/utils/url" +) + +var ( + _ AuthStrategy = &QueryAuthStrategy{} +) + +// QueryAuthStrategy is a strategy for query auth +type QueryAuthStrategy struct { + Data *Secret +} + +// NewQueryAuthStrategy creates a new query auth strategy +func NewQueryAuthStrategy(data *Secret) *QueryAuthStrategy { + return &QueryAuthStrategy{Data: data} +} + +// Apply applies the query auth strategy to the request +func (s *QueryAuthStrategy) Apply(req *http.Request) { + q := urlutil.NewOrderedParams() + q.Decode(req.URL.RawQuery) + for _, p := range s.Data.Params { + q.Add(p.Key, p.Value) + } + req.URL.RawQuery = q.Encode() +} + +// ApplyOnRR applies the query auth strategy to the retryable request +func (s *QueryAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + q := urlutil.NewOrderedParams() + q.Decode(req.Request.URL.RawQuery) + for _, p := range s.Data.Params { + q.Add(p.Key, p.Value) + } + req.Request.URL.RawQuery = q.Encode() +} diff --git a/common/authprovider/authx/strategy.go b/common/authprovider/authx/strategy.go new file mode 100644 index 00000000..35e42e35 --- /dev/null +++ b/common/authprovider/authx/strategy.go @@ -0,0 +1,16 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" +) + +// AuthStrategy is an interface for auth strategies +// basic auth , bearer token, headers, cookies, query +type AuthStrategy interface { + // Apply applies the strategy to the request + Apply(*http.Request) + // ApplyOnRR applies the strategy to the retryable request + ApplyOnRR(*retryablehttp.Request) +} diff --git a/common/authprovider/authx/strategy_test.go b/common/authprovider/authx/strategy_test.go new file mode 100644 index 00000000..01a8a9f5 --- /dev/null +++ b/common/authprovider/authx/strategy_test.go @@ -0,0 +1,217 @@ +package authx + +import ( + "net/http" + "testing" + + "github.com/projectdiscovery/retryablehttp-go" +) + +func TestBasicAuthStrategy(t *testing.T) { + secret := &Secret{ + Username: "user", + Password: "pass", + } + strategy := NewBasicAuthStrategy(secret) + + t.Run("Apply", func(t *testing.T) { + req, _ := http.NewRequest("GET", "http://example.com", nil) + strategy.Apply(req) + + user, pass, ok := req.BasicAuth() + if !ok { + t.Error("Basic auth not set") + } + if user != "user" { + t.Errorf("Username = %v, want user", user) + } + if pass != "pass" { + t.Errorf("Password = %v, want pass", pass) + } + }) + + t.Run("ApplyOnRR", func(t *testing.T) { + req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil) + strategy.ApplyOnRR(req) + + user, pass, ok := req.BasicAuth() + if !ok { + t.Error("Basic auth not set") + } + if user != "user" { + t.Errorf("Username = %v, want user", user) + } + if pass != "pass" { + t.Errorf("Password = %v, want pass", pass) + } + }) +} + +func TestBearerTokenAuthStrategy(t *testing.T) { + secret := &Secret{ + Token: "mytoken123", + } + strategy := NewBearerTokenAuthStrategy(secret) + + t.Run("Apply", func(t *testing.T) { + req, _ := http.NewRequest("GET", "http://example.com", nil) + strategy.Apply(req) + + auth := req.Header.Get("Authorization") + expected := "Bearer mytoken123" + if auth != expected { + t.Errorf("Authorization = %v, want %v", auth, expected) + } + }) + + t.Run("ApplyOnRR", func(t *testing.T) { + req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil) + strategy.ApplyOnRR(req) + + auth := req.Header.Get("Authorization") + expected := "Bearer mytoken123" + if auth != expected { + t.Errorf("Authorization = %v, want %v", auth, expected) + } + }) +} + +func TestHeadersAuthStrategy(t *testing.T) { + // Headers strategy preserves exact casing, so use exact key names + secret := &Secret{ + Headers: []KV{ + {Key: "X-API-Key", Value: "secret123"}, + {Key: "X-Custom", Value: "value"}, + }, + } + strategy := NewHeadersAuthStrategy(secret) + + t.Run("Apply", func(t *testing.T) { + req, _ := http.NewRequest("GET", "http://example.com", nil) + strategy.Apply(req) + + // Use direct map access since headers preserve exact casing + //nolint + if got := req.Header["X-API-Key"]; len(got) == 0 || got[0] != "secret123" { + t.Errorf("X-API-Key = %v, want [secret123]", got) + } + if got := req.Header["X-Custom"]; len(got) == 0 || got[0] != "value" { + t.Errorf("X-Custom = %v, want [value]", got) + } + }) + + t.Run("ApplyOnRR", func(t *testing.T) { + req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil) + strategy.ApplyOnRR(req) + + // Use direct map access since headers preserve exact casing + //nolint + if got := req.Header["X-API-Key"]; len(got) == 0 || got[0] != "secret123" { + t.Errorf("X-API-Key = %v, want [secret123]", got) + } + if got := req.Header["X-Custom"]; len(got) == 0 || got[0] != "value" { + t.Errorf("X-Custom = %v, want [value]", got) + } + }) +} + +func TestCookiesAuthStrategy(t *testing.T) { + secret := &Secret{ + Cookies: []Cookie{ + {Key: "session", Value: "abc123"}, + {Key: "auth", Value: "xyz789"}, + }, + } + strategy := NewCookiesAuthStrategy(secret) + + t.Run("Apply", func(t *testing.T) { + req, _ := http.NewRequest("GET", "http://example.com", nil) + strategy.Apply(req) + + cookies := req.Cookies() + if len(cookies) != 2 { + t.Errorf("Expected 2 cookies, got %d", len(cookies)) + } + + found := make(map[string]string) + for _, c := range cookies { + found[c.Name] = c.Value + } + if found["session"] != "abc123" { + t.Errorf("session cookie = %v, want abc123", found["session"]) + } + if found["auth"] != "xyz789" { + t.Errorf("auth cookie = %v, want xyz789", found["auth"]) + } + }) + + t.Run("ApplyOnRR replaces existing cookies", func(t *testing.T) { + req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil) + // Add existing cookie that should be replaced + req.AddCookie(&http.Cookie{Name: "session", Value: "old_value"}) + // Add existing cookie that should be kept + req.AddCookie(&http.Cookie{Name: "other", Value: "keep_me"}) + + strategy.ApplyOnRR(req) + + cookies := req.Cookies() + found := make(map[string]string) + for _, c := range cookies { + found[c.Name] = c.Value + } + + // New cookie values should override old ones + if found["session"] != "abc123" { + t.Errorf("session cookie = %v, want abc123", found["session"]) + } + if found["auth"] != "xyz789" { + t.Errorf("auth cookie = %v, want xyz789", found["auth"]) + } + // Existing non-replaced cookie should be preserved + if found["other"] != "keep_me" { + t.Errorf("other cookie = %v, want keep_me", found["other"]) + } + }) +} + +func TestQueryAuthStrategy(t *testing.T) { + secret := &Secret{ + Params: []KV{ + {Key: "api_key", Value: "secret123"}, + {Key: "token", Value: "abc"}, + }, + } + strategy := NewQueryAuthStrategy(secret) + + t.Run("Apply", func(t *testing.T) { + req, _ := http.NewRequest("GET", "http://example.com/path?existing=value", nil) + strategy.Apply(req) + + query := req.URL.Query() + if got := query.Get("api_key"); got != "secret123" { + t.Errorf("api_key = %v, want secret123", got) + } + if got := query.Get("token"); got != "abc" { + t.Errorf("token = %v, want abc", got) + } + if got := query.Get("existing"); got != "value" { + t.Errorf("existing = %v, want value", got) + } + }) + + t.Run("ApplyOnRR", func(t *testing.T) { + req, _ := retryablehttp.NewRequest("GET", "http://example.com/path?existing=value", nil) + strategy.ApplyOnRR(req) + + query := req.Request.URL.Query() + if got := query.Get("api_key"); got != "secret123" { + t.Errorf("api_key = %v, want secret123", got) + } + if got := query.Get("token"); got != "abc" { + t.Errorf("token = %v, want abc", got) + } + if got := query.Get("existing"); got != "value" { + t.Errorf("existing = %v, want value", got) + } + }) +} diff --git a/common/authprovider/file.go b/common/authprovider/file.go new file mode 100644 index 00000000..ca6d6073 --- /dev/null +++ b/common/authprovider/file.go @@ -0,0 +1,114 @@ +package authprovider + +import ( + "net" + "net/url" + "regexp" + "strings" + + "github.com/projectdiscovery/httpx/common/authprovider/authx" + "github.com/projectdiscovery/utils/errkit" + urlutil "github.com/projectdiscovery/utils/url" +) + +// FileAuthProvider is an auth provider for file based auth +// it accepts a secrets file and returns its provider +type FileAuthProvider struct { + Path string + store *authx.Authx + compiled map[*regexp.Regexp][]authx.AuthStrategy + domains map[string][]authx.AuthStrategy +} + +// NewFileAuthProvider creates a new file based auth provider +func NewFileAuthProvider(path string) (AuthProvider, error) { + store, err := authx.GetAuthDataFromFile(path) + if err != nil { + return nil, err + } + if len(store.Secrets) == 0 { + return nil, ErrNoSecrets + } + for _, secret := range store.Secrets { + if err := secret.Validate(); err != nil { + errorErr := errkit.FromError(err) + errorErr.Msgf("invalid secret in file: %s", path) + return nil, errorErr + } + } + f := &FileAuthProvider{Path: path, store: store} + f.init() + return f, nil +} + +// init initializes the file auth provider +func (f *FileAuthProvider) init() { + for _, _secret := range f.store.Secrets { + secret := _secret // capture loop variable for use in GetStrategy() + if len(secret.DomainsRegex) > 0 { + for _, domain := range secret.DomainsRegex { + if f.compiled == nil { + f.compiled = make(map[*regexp.Regexp][]authx.AuthStrategy) + } + compiled, err := regexp.Compile(domain) + if err != nil { + continue + } + + if ss, ok := f.compiled[compiled]; ok { + f.compiled[compiled] = append(ss, secret.GetStrategy()) + } else { + f.compiled[compiled] = []authx.AuthStrategy{secret.GetStrategy()} + } + } + } + for _, domain := range secret.Domains { + if f.domains == nil { + f.domains = make(map[string][]authx.AuthStrategy) + } + domain = strings.TrimSpace(domain) + domain = strings.TrimSuffix(domain, ":80") + domain = strings.TrimSuffix(domain, ":443") + if ss, ok := f.domains[domain]; ok { + f.domains[domain] = append(ss, secret.GetStrategy()) + } else { + f.domains[domain] = []authx.AuthStrategy{secret.GetStrategy()} + } + } + } +} + +// LookupAddr looks up a given domain/address and returns appropriate auth strategy +func (f *FileAuthProvider) LookupAddr(addr string) []authx.AuthStrategy { + var strategies []authx.AuthStrategy + + if strings.Contains(addr, ":") { + // strip default ports (80/443) for consistent domain matching + host, port, err := net.SplitHostPort(addr) + if err == nil && (port == "80" || port == "443") { + addr = host + } + } + for domain, strategy := range f.domains { + if strings.EqualFold(domain, addr) { + strategies = append(strategies, strategy...) + } + } + for compiled, strategy := range f.compiled { + if compiled.MatchString(addr) { + strategies = append(strategies, strategy...) + } + } + + return strategies +} + +// LookupURL looks up a given URL and returns appropriate auth strategy +func (f *FileAuthProvider) LookupURL(u *url.URL) []authx.AuthStrategy { + return f.LookupAddr(u.Host) +} + +// LookupURLX looks up a given URL and returns appropriate auth strategy +func (f *FileAuthProvider) LookupURLX(u *urlutil.URL) []authx.AuthStrategy { + return f.LookupAddr(u.Host) +} diff --git a/common/authprovider/interface.go b/common/authprovider/interface.go new file mode 100644 index 00000000..981496fc --- /dev/null +++ b/common/authprovider/interface.go @@ -0,0 +1,53 @@ +// TODO: This package should be abstracted out to projectdiscovery/utils +// so it can be shared between httpx, nuclei, and other tools. +package authprovider + +import ( + "fmt" + "net/url" + + "github.com/projectdiscovery/httpx/common/authprovider/authx" + urlutil "github.com/projectdiscovery/utils/url" +) + +var ( + ErrNoSecrets = fmt.Errorf("no secrets in given provider") +) + +var ( + _ AuthProvider = &FileAuthProvider{} +) + +// AuthProvider is an interface for auth providers +// It implements a data structure suitable for quick lookup and retrieval +// of auth strategies +type AuthProvider interface { + // LookupAddr looks up a given domain/address and returns appropriate auth strategy + // for it (accepted inputs are scanme.sh or scanme.sh:443) + LookupAddr(string) []authx.AuthStrategy + // LookupURL looks up a given URL and returns appropriate auth strategy + // it accepts a valid url struct and returns the auth strategy + LookupURL(*url.URL) []authx.AuthStrategy + // LookupURLX looks up a given URL and returns appropriate auth strategy + // it accepts pd url struct (i.e urlutil.URL) and returns the auth strategy + LookupURLX(*urlutil.URL) []authx.AuthStrategy +} + +// AuthProviderOptions contains options for the auth provider +type AuthProviderOptions struct { + // File based auth provider options + SecretsFiles []string +} + +// NewAuthProvider creates a new auth provider from the given options +func NewAuthProvider(options *AuthProviderOptions) (AuthProvider, error) { + var providers []AuthProvider + for _, file := range options.SecretsFiles { + provider, err := NewFileAuthProvider(file) + if err != nil { + return nil, err + } + providers = append(providers, provider) + } + return NewMultiAuthProvider(providers...), nil +} diff --git a/common/authprovider/multi.go b/common/authprovider/multi.go new file mode 100644 index 00000000..0ef00551 --- /dev/null +++ b/common/authprovider/multi.go @@ -0,0 +1,50 @@ +package authprovider + +import ( + "net/url" + + "github.com/projectdiscovery/httpx/common/authprovider/authx" + urlutil "github.com/projectdiscovery/utils/url" +) + +// MultiAuthProvider is a convenience wrapper for multiple auth providers +// it returns the first matching auth strategy for a given domain +// if there are multiple auth strategies for a given domain, it returns the first one +type MultiAuthProvider struct { + Providers []AuthProvider +} + +// NewMultiAuthProvider creates a new multi auth provider +func NewMultiAuthProvider(providers ...AuthProvider) AuthProvider { + return &MultiAuthProvider{Providers: providers} +} + +func (m *MultiAuthProvider) LookupAddr(host string) []authx.AuthStrategy { + for _, provider := range m.Providers { + strategy := provider.LookupAddr(host) + if len(strategy) > 0 { + return strategy + } + } + return nil +} + +func (m *MultiAuthProvider) LookupURL(u *url.URL) []authx.AuthStrategy { + for _, provider := range m.Providers { + strategy := provider.LookupURL(u) + if len(strategy) > 0 { + return strategy + } + } + return nil +} + +func (m *MultiAuthProvider) LookupURLX(u *urlutil.URL) []authx.AuthStrategy { + for _, provider := range m.Providers { + strategy := provider.LookupURLX(u) + if len(strategy) > 0 { + return strategy + } + } + return nil +} diff --git a/common/authprovider/provider_test.go b/common/authprovider/provider_test.go new file mode 100644 index 00000000..a29cde51 --- /dev/null +++ b/common/authprovider/provider_test.go @@ -0,0 +1,256 @@ +package authprovider + +import ( + "net/url" + "os" + "path/filepath" + "testing" + + urlutil "github.com/projectdiscovery/utils/url" +) + +func createTestSecretsFile(t *testing.T, content string) string { + t.Helper() + tmpDir := t.TempDir() + filePath := filepath.Join(tmpDir, "secrets.yaml") + err := os.WriteFile(filePath, []byte(content), 0644) + if err != nil { + t.Fatalf("Failed to create test secrets file: %v", err) + } + return filePath +} + +func TestFileAuthProviderLookupAddr(t *testing.T) { + content := `id: test +info: + name: test +static: + - type: BasicAuth + domains: + - example.com + - api.example.com:443 + username: user + password: pass + - type: BearerToken + domains-regex: + - ".*\\.test\\.com" + token: regextoken +` + filePath := createTestSecretsFile(t, content) + provider, err := NewFileAuthProvider(filePath) + if err != nil { + t.Fatalf("NewFileAuthProvider() error = %v", err) + } + + tests := []struct { + name string + addr string + wantCount int + }{ + { + name: "exact match", + addr: "example.com", + wantCount: 1, + }, + { + name: "exact match case insensitive", + addr: "EXAMPLE.COM", + wantCount: 1, + }, + { + name: "with port 443 normalized", + addr: "example.com:443", + wantCount: 1, + }, + { + name: "with port 80 normalized", + addr: "example.com:80", + wantCount: 1, + }, + { + name: "subdomain exact match", + addr: "api.example.com", + wantCount: 1, + }, + { + name: "regex match", + addr: "foo.test.com", + wantCount: 1, + }, + { + name: "regex match subdomain", + addr: "bar.baz.test.com", + wantCount: 1, + }, + { + name: "no match", + addr: "unknown.com", + wantCount: 0, + }, + { + name: "non-standard port not normalized", + addr: "example.com:8080", + wantCount: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + strategies := provider.LookupAddr(tt.addr) + if len(strategies) != tt.wantCount { + t.Errorf("LookupAddr(%q) returned %d strategies, want %d", tt.addr, len(strategies), tt.wantCount) + } + }) + } +} + +func TestFileAuthProviderLookupURL(t *testing.T) { + content := `id: test +info: + name: test +static: + - type: BasicAuth + domains: + - example.com + username: user + password: pass +` + filePath := createTestSecretsFile(t, content) + provider, err := NewFileAuthProvider(filePath) + if err != nil { + t.Fatalf("NewFileAuthProvider() error = %v", err) + } + + t.Run("LookupURL", func(t *testing.T) { + u, _ := url.Parse("https://example.com/path") + strategies := provider.LookupURL(u) + if len(strategies) != 1 { + t.Errorf("LookupURL() returned %d strategies, want 1", len(strategies)) + } + }) + + t.Run("LookupURLX", func(t *testing.T) { + u, _ := urlutil.Parse("https://example.com/path") + strategies := provider.LookupURLX(u) + if len(strategies) != 1 { + t.Errorf("LookupURLX() returned %d strategies, want 1", len(strategies)) + } + }) +} + +func TestMultiAuthProvider(t *testing.T) { + content1 := `id: test1 +info: + name: test1 +static: + - type: BasicAuth + domains: + - first.com + username: user1 + password: pass1 +` + content2 := `id: test2 +info: + name: test2 +static: + - type: BearerToken + domains: + - second.com + token: token2 +` + filePath1 := createTestSecretsFile(t, content1) + provider1, err := NewFileAuthProvider(filePath1) + if err != nil { + t.Fatalf("NewFileAuthProvider() error = %v", err) + } + + // Create second file in different temp dir + tmpDir2 := t.TempDir() + filePath2 := filepath.Join(tmpDir2, "secrets2.yaml") + err = os.WriteFile(filePath2, []byte(content2), 0644) + if err != nil { + t.Fatalf("Failed to create test secrets file: %v", err) + } + provider2, err := NewFileAuthProvider(filePath2) + if err != nil { + t.Fatalf("NewFileAuthProvider() error = %v", err) + } + + multi := NewMultiAuthProvider(provider1, provider2) + + tests := []struct { + name string + addr string + wantCount int + }{ + { + name: "match first provider", + addr: "first.com", + wantCount: 1, + }, + { + name: "match second provider", + addr: "second.com", + wantCount: 1, + }, + { + name: "no match", + addr: "third.com", + wantCount: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + strategies := multi.LookupAddr(tt.addr) + if len(strategies) != tt.wantCount { + t.Errorf("LookupAddr(%q) returned %d strategies, want %d", tt.addr, len(strategies), tt.wantCount) + } + }) + } +} + +func TestNewFileAuthProviderErrors(t *testing.T) { + tests := []struct { + name string + content string + wantErr bool + }{ + { + name: "empty secrets", + content: `id: test`, + wantErr: true, + }, + { + name: "invalid secret type", + content: `id: test +static: + - type: InvalidType + domains: + - example.com +`, + wantErr: true, + }, + { + name: "missing required field", + content: `id: test +static: + - type: BasicAuth + domains: + - example.com + username: user +`, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + filePath := createTestSecretsFile(t, tt.content) + _, err := NewFileAuthProvider(filePath) + if (err != nil) != tt.wantErr { + t.Errorf("NewFileAuthProvider() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 39d1ce38..039f4c4c 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -235,6 +235,14 @@ get_response: resp.Headers = httpresp.Header.Clone() + if h.Options.MaxResponseBodySizeToRead > 0 { + httpresp.Body = io.NopCloser(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead)) + defer func() { + _, _ = io.Copy(io.Discard, httpresp.Body) + _ = httpresp.Body.Close() + }() + } + // httputil.DumpResponse does not handle websockets headers, rawResp, err := pdhttputil.DumpResponseHeadersAndRaw(httpresp) if err != nil { diff --git a/common/httpx/option.go b/common/httpx/option.go index b64cfd39..fb108729 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -5,10 +5,19 @@ import ( "strings" "time" + "github.com/dustin/go-humanize" "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/networkpolicy" ) +// DefaultMaxResponseBodySize is the default maximum response body size +var DefaultMaxResponseBodySize int64 + +func init() { + maxResponseBodySize, _ := humanize.ParseBytes("512Mb") + DefaultMaxResponseBodySize = int64(maxResponseBodySize) +} + // Options contains configuration options for the client type Options struct { RandomAgent bool @@ -66,7 +75,7 @@ var DefaultOptions = Options{ Unsafe: false, CdnCheck: "true", ExcludeCdn: false, - MaxResponseBodySizeToRead: 1024 * 1024 * 10, + MaxResponseBodySizeToRead: DefaultMaxResponseBodySize, // VHOSTs options VHostIgnoreStatusCode: false, VHostIgnoreContentLength: true, diff --git a/common/inputformats/burp.go b/common/inputformats/burp.go new file mode 100644 index 00000000..8af34327 --- /dev/null +++ b/common/inputformats/burp.go @@ -0,0 +1,44 @@ +package inputformats + +import ( + "io" + + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/seh-msft/burpxml" +) + +// BurpFormat is a Burp Suite XML file parser +type BurpFormat struct{} + +// NewBurpFormat creates a new Burp XML file parser +func NewBurpFormat() *BurpFormat { + return &BurpFormat{} +} + +var _ Format = &BurpFormat{} + +// Name returns the name of the format +func (b *BurpFormat) Name() string { + return "burp" +} + +// Parse parses the Burp XML input and calls the provided callback +// function for each URL it discovers. +func (b *BurpFormat) Parse(input io.Reader, callback func(url string) bool) error { + items, err := burpxml.Parse(input, true) + if err != nil { + return errors.Wrap(err, "could not parse burp xml") + } + + for i, item := range items.Items { + if item.Url == "" { + gologger.Debug().Msgf("Skipping burp item %d: empty URL", i) + continue + } + if !callback(item.Url) { + break + } + } + return nil +} diff --git a/common/inputformats/burp_test.go b/common/inputformats/burp_test.go new file mode 100644 index 00000000..ad2db527 --- /dev/null +++ b/common/inputformats/burp_test.go @@ -0,0 +1,169 @@ +package inputformats + +import ( + "strings" + "testing" +) + +func TestBurpFormat_Name(t *testing.T) { + b := NewBurpFormat() + if b.Name() != "burp" { + t.Errorf("Expected name 'burp', got '%s'", b.Name()) + } +} + +func TestBurpFormat_Parse(t *testing.T) { + burpXML := ` + + + + + example.com + 80 + http + + + null + + 200 + 100 + HTML + + + + + + + example.com + 443 + https + + + null + + 200 + 100 + JSON + + + +` + + b := NewBurpFormat() + var urls []string + + err := b.Parse(strings.NewReader(burpXML), func(url string) bool { + urls = append(urls, url) + return true + }) + + if err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + if len(urls) != 2 { + t.Errorf("Expected 2 URLs, got %d", len(urls)) + } + + expectedURLs := []string{"http://example.com/path1", "https://example.com/path2"} + if len(urls) != len(expectedURLs) { + t.Fatalf("Expected %d URLs, got %d: %v", len(expectedURLs), len(urls), urls) + } + for i, expected := range expectedURLs { + if urls[i] != expected { + t.Errorf("Expected URL %d to be '%s', got '%s'", i, expected, urls[i]) + } + } +} + +func TestBurpFormat_ParseEmpty(t *testing.T) { + burpXML := ` + +` + + b := NewBurpFormat() + var urls []string + + err := b.Parse(strings.NewReader(burpXML), func(url string) bool { + urls = append(urls, url) + return true + }) + + if err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + if len(urls) != 0 { + t.Errorf("Expected 0 URLs, got %d", len(urls)) + } +} + +func TestBurpFormat_ParseStopEarly(t *testing.T) { + burpXML := ` + + + + example.com + 80 + http + + + null + + 200 + 100 + HTML + + + + + + example.com + 80 + http + + + null + + 200 + 100 + HTML + + + +` + + b := NewBurpFormat() + var urls []string + + err := b.Parse(strings.NewReader(burpXML), func(url string) bool { + urls = append(urls, url) + return false // stop after first + }) + + if err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + if len(urls) != 1 { + t.Errorf("Expected 1 URL (stopped early), got %d", len(urls)) + } +} + +func TestBurpFormat_ParseMalformed(t *testing.T) { + malformedXML := ` + + + + +` + + b := NewBurpFormat() + err := b.Parse(strings.NewReader(malformedXML), func(url string) bool { + return true + }) + + if err == nil { + t.Error("Expected error for malformed XML, got nil") + } +} diff --git a/common/inputformats/formats.go b/common/inputformats/formats.go new file mode 100644 index 00000000..31a25a38 --- /dev/null +++ b/common/inputformats/formats.go @@ -0,0 +1,41 @@ +// TODO: This package should be abstracted out to projectdiscovery/utils +// so it can be shared between httpx, nuclei, and other tools. +package inputformats + +import ( + "io" + "strings" +) + +// Format is an interface implemented by all input formats +type Format interface { + // Name returns the name of the format + Name() string + // Parse parses the input and calls the provided callback + // function for each URL it discovers. + Parse(input io.Reader, callback func(url string) bool) error +} + +// Supported formats +var formats = []Format{ + NewBurpFormat(), +} + +// GetFormat returns the format by name +func GetFormat(name string) Format { + for _, f := range formats { + if strings.EqualFold(f.Name(), name) { + return f + } + } + return nil +} + +// SupportedFormats returns a comma-separated list of supported format names +func SupportedFormats() string { + var names []string + for _, f := range formats { + names = append(names, f.Name()) + } + return strings.Join(names, ", ") +} diff --git a/common/inputformats/formats_test.go b/common/inputformats/formats_test.go new file mode 100644 index 00000000..aefe05d2 --- /dev/null +++ b/common/inputformats/formats_test.go @@ -0,0 +1,43 @@ +package inputformats + +import ( + "strings" + "testing" +) + +func TestGetFormat(t *testing.T) { + tests := []struct { + name string + input string + wantNil bool + wantName string + }{ + {"burp lowercase", "burp", false, "burp"}, + {"burp uppercase", "BURP", false, "burp"}, + {"burp mixed case", "Burp", false, "burp"}, + {"invalid format", "invalid", true, ""}, + {"empty string", "", true, ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GetFormat(tt.input) + if tt.wantNil && got != nil { + t.Errorf("GetFormat(%q) = %v, want nil", tt.input, got) + } + if !tt.wantNil && got == nil { + t.Errorf("GetFormat(%q) = nil, want non-nil", tt.input) + } + if !tt.wantNil && got != nil && got.Name() != tt.wantName { + t.Errorf("GetFormat(%q).Name() = %q, want %q", tt.input, got.Name(), tt.wantName) + } + }) + } +} + +func TestSupportedFormats(t *testing.T) { + supported := SupportedFormats() + if !strings.Contains(supported, "burp") { + t.Errorf("SupportedFormats() = %q, expected to contain 'burp'", supported) + } +} diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go index 5033e1cb..294dc9d8 100644 --- a/common/stringz/stringz.go +++ b/common/stringz/stringz.go @@ -85,9 +85,10 @@ func AddURLDefaultPort(rawURL string) string { } // Force default port to be added if not present if u.Port() == "" { - if u.Scheme == urlutil.HTTP { + switch u.Scheme { + case urlutil.HTTP: u.UpdatePort("80") - } else if u.Scheme == urlutil.HTTPS { + case urlutil.HTTPS: u.UpdatePort("443") } } diff --git a/go.mod b/go.mod index 686a8717..00baad81 100644 --- a/go.mod +++ b/go.mod @@ -19,24 +19,24 @@ require ( github.com/miekg/dns v1.1.68 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/asnmap v1.1.1 - github.com/projectdiscovery/cdncheck v1.2.13 + github.com/projectdiscovery/cdncheck v1.2.18 github.com/projectdiscovery/clistats v0.1.1 - github.com/projectdiscovery/dsl v0.8.7 - github.com/projectdiscovery/fastdialer v0.4.19 + github.com/projectdiscovery/dsl v0.8.12 + github.com/projectdiscovery/fastdialer v0.5.3 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 github.com/projectdiscovery/goflags v0.1.74 - github.com/projectdiscovery/gologger v1.1.63 - github.com/projectdiscovery/hmap v0.0.98 + github.com/projectdiscovery/gologger v1.1.67 + github.com/projectdiscovery/hmap v0.0.99 github.com/projectdiscovery/mapcidr v1.1.97 - github.com/projectdiscovery/networkpolicy v0.1.31 - github.com/projectdiscovery/ratelimit v0.0.82 + github.com/projectdiscovery/networkpolicy v0.1.33 + github.com/projectdiscovery/ratelimit v0.0.83 github.com/projectdiscovery/rawhttp v0.1.90 - github.com/projectdiscovery/retryablehttp-go v1.1.0 + github.com/projectdiscovery/retryablehttp-go v1.3.3 github.com/projectdiscovery/tlsx v1.2.2 - github.com/projectdiscovery/useragent v0.0.105 - github.com/projectdiscovery/utils v0.7.3 - github.com/projectdiscovery/wappalyzergo v0.2.58 + github.com/projectdiscovery/useragent v0.0.106 + github.com/projectdiscovery/utils v0.8.0 + github.com/projectdiscovery/wappalyzergo v0.2.63 github.com/rs/xid v1.6.0 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.11.1 @@ -44,16 +44,20 @@ require ( go.etcd.io/bbolt v1.4.0 // indirect go.uber.org/multierr v1.11.0 golang.org/x/exp v0.0.0-20250911091902-df9299821621 - golang.org/x/net v0.47.0 - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 + golang.org/x/net v0.48.0 + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.33.0 ) require ( github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0 + github.com/dustin/go-humanize v1.0.1 github.com/go-viper/mapstructure/v2 v2.4.0 github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 - github.com/weppos/publicsuffix-go v0.50.1 + github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 + github.com/seh-msft/burpxml v1.0.1 + github.com/weppos/publicsuffix-go v0.50.2 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -62,16 +66,16 @@ require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect - github.com/STARRY-S/zip v0.2.1 // indirect + github.com/STARRY-S/zip v0.2.3 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect - github.com/andybalholm/brotli v1.1.1 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bodgit/plumbing v1.3.0 // indirect - github.com/bodgit/sevenzip v1.6.0 // indirect + github.com/bodgit/sevenzip v1.6.1 // indirect github.com/bodgit/windows v1.0.1 // indirect github.com/brianvoe/gofakeit/v7 v7.2.1 // indirect github.com/charmbracelet/glamour v0.8.0 // indirect @@ -83,6 +87,7 @@ require ( github.com/cnf/structhash v0.0.0-20250313080605-df4c6cc74a9a // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/djherbis/times v1.6.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect @@ -99,37 +104,38 @@ require ( github.com/gorilla/css v1.0.1 // indirect github.com/gosimple/slug v1.15.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/iangcarroll/cookiemonster v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kataras/jwt v0.1.10 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.2 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kljensen/snowball v0.8.0 // indirect + github.com/logrusorgru/aurora/v4 v4.0.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect - github.com/mholt/archives v0.1.0 // indirect + github.com/mholt/archives v0.1.5 // indirect + github.com/mikelolasagasti/xz v1.0.1 // indirect + github.com/minio/minlz v1.0.1 // indirect github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect - github.com/nwaples/rardecode/v2 v2.2.0 // indirect - github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/nwaples/rardecode/v2 v2.2.2 // indirect + github.com/pierrec/lz4/v4 v4.1.23 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/freeport v0.0.7 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect - github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/retryabledns v1.0.110 // indirect + github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 // indirect + github.com/projectdiscovery/retryabledns v1.0.112 // indirect github.com/refraction-networking/utls v1.7.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect @@ -137,9 +143,9 @@ require ( github.com/sashabaranov/go-openai v1.37.0 // indirect github.com/shirou/gopsutil/v3 v3.24.2 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect - github.com/sorairolake/lzip-go v0.3.5 // indirect + github.com/sorairolake/lzip-go v0.3.8 // indirect + github.com/spf13/afero v1.15.0 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect - github.com/therootcompany/xz v1.0.1 // indirect github.com/tidwall/btree v1.7.0 // indirect github.com/tidwall/buntdb v1.3.1 // indirect github.com/tidwall/gjson v1.18.0 // indirect @@ -164,14 +170,12 @@ require ( github.com/zcalusic/sysinfo v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/mod v0.29.0 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/mod v0.31.0 // indirect golang.org/x/oauth2 v0.28.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/term v0.37.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/term v0.38.0 // indirect golang.org/x/time v0.11.0 // indirect - golang.org/x/tools v0.38.0 // indirect - gopkg.in/djherbis/times.v1 v1.3.0 // indirect + golang.org/x/tools v0.40.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index df546100..63265e06 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0g github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw= github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ= github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE= -github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= -github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= +github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4= +github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi78= @@ -48,8 +48,8 @@ github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46 github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= -github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= @@ -66,8 +66,8 @@ github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWk github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= -github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A= -github.com/bodgit/sevenzip v1.6.0/go.mod h1:zOBh9nJUof7tcrlqJFv1koWRrhz3LbDbUNngkuZxLMc= +github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4= +github.com/bodgit/sevenzip v1.6.1/go.mod h1:GVoYQbEVbOGT8n2pfqCIMRUaRjQ8F9oSqoBEqZh5fQ8= github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/brianvoe/gofakeit/v7 v7.2.1 h1:AGojgaaCdgq4Adzrd2uWdbGNDyX6MWNhHdQBraNfOHI= @@ -109,6 +109,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= +github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -116,6 +118,8 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= @@ -204,11 +208,6 @@ github.com/gosimple/slug v1.15.0 h1:wRZHsRrRcs6b0XnxMUBM6WK1U1Vg5B0R7VkIf1Xzobo= github.com/gosimple/slug v1.15.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -237,8 +236,8 @@ github.com/kataras/jwt v0.1.10 h1:GBXOF9RVInDPhCFBiDumRG9Tt27l7ugLeLo8HL5SeKQ= github.com/kataras/jwt v0.1.10/go.mod h1:xkimAtDhU/aGlQqjwvgtg+VyuPwMiyZHaY8LJRh0mYo= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -255,6 +254,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= +github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -269,13 +270,17 @@ github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6T github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= -github.com/mholt/archives v0.1.0 h1:FacgJyrjiuyomTuNA92X5GyRBRZjE43Y/lrzKIlF35Q= -github.com/mholt/archives v0.1.0/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I= +github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ= +github.com/mholt/archives v0.1.5/go.mod h1:3TPMmBLPsgszL+1As5zECTuKwKvIfj6YcwWPpeTAXF4= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA= github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps= +github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0= +github.com/mikelolasagasti/xz v1.0.1/go.mod h1:muAirjiOUxPRXwm9HdDtB3uoRPrGnL85XHtokL9Hcgc= +github.com/minio/minlz v1.0.1 h1:OUZUzXcib8diiX+JYxyRLIdomyZYzHct6EShOKtQY2A= +github.com/minio/minlz v1.0.1/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -291,8 +296,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= -github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A= -github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= +github.com/nwaples/rardecode/v2 v2.2.2 h1:/5oL8dzYivRM/tqX9VcTSWfbpwcbwKG1QtSJr3b3KcU= +github.com/nwaples/rardecode/v2 v2.2.2/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -304,8 +309,8 @@ github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= -github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.23 h1:oJE7T90aYBGtFNrI8+KbETnPymobAhzRrR8Mu8n1yfU= +github.com/pierrec/lz4/v4 v4.1.23/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -315,16 +320,18 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kID2iwsDqI= github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60= +github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 h1:UCZRqs1BP1wsvhCwQxfIQc7NJcXGBhQvAnEw3awhsng= +github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.2.13 h1:6zs4Mn8JV3yKyMoAr857Hf2NLvyOMpOfqCCT2V2OI1Q= -github.com/projectdiscovery/cdncheck v1.2.13/go.mod h1:/OhuZ9T25yXSqU6+oWvmVQ3QFvtew/Tp03u0jM+NJBE= +github.com/projectdiscovery/cdncheck v1.2.18 h1:3hChFW3/nVF28ktnXqsP/3DlUTghVs8Xg4+XM/Ui87s= +github.com/projectdiscovery/cdncheck v1.2.18/go.mod h1:RRA4KOiUTBhkk2tImdoxqPpD0fB5C9rBP7W0r+ji9Cg= github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE= github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= -github.com/projectdiscovery/dsl v0.8.7 h1:LZpSOgET86X92t7E3f2SY8YZQhGiw7yZ38vSa6pK75s= -github.com/projectdiscovery/dsl v0.8.7/go.mod h1:Dqbtd4nPASPtNpHAweIUhCMLo00S2q6oTIgrZ+kyN+4= -github.com/projectdiscovery/fastdialer v0.4.19 h1:MLHwEGM0x0pyltJaNvAVvwc27bnXdZ5mYr50S/2kMEE= -github.com/projectdiscovery/fastdialer v0.4.19/go.mod h1:HGdVsez+JgJ9/ljXjHRplOqkB7og+nqi0nrNWVNi03o= +github.com/projectdiscovery/dsl v0.8.12 h1:gQL8k5zPok+5JGc7poiXzHCElNY/WnaTKoRB2wI3CYA= +github.com/projectdiscovery/dsl v0.8.12/go.mod h1:pdMfUTNHMxlt6M94CSrCpZ1QObTP44rLqWifMMWW+IA= +github.com/projectdiscovery/fastdialer v0.5.3 h1:Io57Q37ouFzrPK53ZdzK6jsELgqjIMCWcoDs+lRDGMA= +github.com/projectdiscovery/fastdialer v0.5.3/go.mod h1:euoxS1E93LDnl0OnNN0UALedAFF+EehBxyU3z+79l0g= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk= @@ -333,36 +340,36 @@ github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQ github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c= github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= -github.com/projectdiscovery/gologger v1.1.63 h1:oboXTekGtbmo9bPAAzkeCeo4EPHAwAdHmknDSAEo/1c= -github.com/projectdiscovery/gologger v1.1.63/go.mod h1:daez34xaA7LTazBb4t+Ccm9/9Kjvlsu4EY033lQdATA= +github.com/projectdiscovery/gologger v1.1.67 h1:GZU3AjYiJvcwJT5TlfIv+152/TVmaz62Zyn3/wWXlig= +github.com/projectdiscovery/gologger v1.1.67/go.mod h1:35oeQP6wvj58S+o+Km6boED/t786FXQkI0exhFHJbNE= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.98 h1:XxYIi7yJCNiDAKCJXvuY9IBM5O6OgDgx4XHgKxkR4eg= -github.com/projectdiscovery/hmap v0.0.98/go.mod h1:bgN5fuZPJMj2YnAGEEnCypoifCnALJixHEVQszktQIU= -github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= -github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= +github.com/projectdiscovery/hmap v0.0.99 h1:XPfLnD3CUrMqVCIdpK9ozD7Xmp3simx3T+2j4WWhHnU= +github.com/projectdiscovery/hmap v0.0.99/go.mod h1:koyUJi83K5G3w35ZLFXOYZIyYJsO+6hQrgDDN1RBrVE= +github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 h1:eR+0HE//Ciyfwy3HC7fjRyKShSJHYoX2Pv7pPshjK/Q= +github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8QuWs2puwy+2w= github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA= -github.com/projectdiscovery/networkpolicy v0.1.31 h1:mE6iJeYOSql8gps/91vwiztE/kEHe5Im8oUO5Mkj9Zg= -github.com/projectdiscovery/networkpolicy v0.1.31/go.mod h1:5x4rGh4XhnoYl9wACnZyrjDGKIB/bQqxw2KrIM5V+XU= -github.com/projectdiscovery/ratelimit v0.0.82 h1:rtO5SQf5uQFu5zTahTaTcO06OxmG8EIF1qhdFPIyTak= -github.com/projectdiscovery/ratelimit v0.0.82/go.mod h1:z076BrLkBb5yS7uhHNoCTf8X/BvFSGRxwQ8EzEL9afM= +github.com/projectdiscovery/networkpolicy v0.1.33 h1:bVgp+XpLEsQ7ZEJt3UaUqIwhI01MMdt7F2dfIKFQg/w= +github.com/projectdiscovery/networkpolicy v0.1.33/go.mod h1:YAPddAXUc/lhoU85AFdvgOQKx8Qh8r0vzSjexRWk6Yk= +github.com/projectdiscovery/ratelimit v0.0.83 h1:hfb36QvznBrjA4FNfpFE8AYRVBYrfJh8qHVROLQgl54= +github.com/projectdiscovery/ratelimit v0.0.83/go.mod h1:z076BrLkBb5yS7uhHNoCTf8X/BvFSGRxwQ8EzEL9afM= github.com/projectdiscovery/rawhttp v0.1.90 h1:LOSZ6PUH08tnKmWsIwvwv1Z/4zkiYKYOSZ6n+8RFKtw= github.com/projectdiscovery/rawhttp v0.1.90/go.mod h1:VZYAM25UI/wVB3URZ95ZaftgOnsbphxyAw/XnQRRz4Y= -github.com/projectdiscovery/retryabledns v1.0.110 h1:24p1PzWBdfsRnGsBf6ZxXPzvK0sYaL4q/ju4+2OhJzU= -github.com/projectdiscovery/retryabledns v1.0.110/go.mod h1:GFj5HjxfaGrZeoYf79zI/R99XljBNjmOqNvwOqPepRU= -github.com/projectdiscovery/retryablehttp-go v1.1.0 h1:uYp3EnuhhamTwvG41X6q6TAc/SHEO9pw9CBWbRASIQk= -github.com/projectdiscovery/retryablehttp-go v1.1.0/go.mod h1:9DU57ezv5cfZSWw/m5XFDTMjy1yKeMyn1kj35lPlcfM= +github.com/projectdiscovery/retryabledns v1.0.112 h1:4iCiuo6jMnw/pdOZRzBQrbUOUu5tOeuvGupxVV8RDLw= +github.com/projectdiscovery/retryabledns v1.0.112/go.mod h1:xsJTKbo+KGqd7+88z1naEUFJybLH2yjB/zUyOweA7k0= +github.com/projectdiscovery/retryablehttp-go v1.3.3 h1:+hj9TwUegVpjFsOpJ+JRNS4iYATmNw4Y52BnjLD1pKU= +github.com/projectdiscovery/retryablehttp-go v1.3.3/go.mod h1:zhdi48RdqMay0S2mhNmG2uHDXVJw3xGMLZ2XbfRprQs= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.2.2 h1:Y96QBqeD2anpzEtBl4kqNbwzXh2TrzJuXfgiBLvK+SE= github.com/projectdiscovery/tlsx v1.2.2/go.mod h1:ZJl9F1sSl0sdwE+lR0yuNHVX4Zx6tCSTqnNxnHCFZB4= -github.com/projectdiscovery/useragent v0.0.105 h1:yFGFTfWZ/RZP5XbGRJtvKcbRNnlGI6xfPRXHb8DWdhg= -github.com/projectdiscovery/useragent v0.0.105/go.mod h1:jWG1BD2yu8EC3olt6Amke7BiyLkXzpErI7Jtzr/tWZM= -github.com/projectdiscovery/utils v0.7.3 h1:kX+77AA58yK6EZgkTRJEnK9V/7AZYzlXdcu/o/kJhFs= -github.com/projectdiscovery/utils v0.7.3/go.mod h1:uDdQ3/VWomai98l+a3Ye/srDXdJ4xUIar/mSXlQ9gBM= -github.com/projectdiscovery/wappalyzergo v0.2.58 h1:MpdnqvozBuFZ9lnrkAUJI7mqOTAoMcDMZ3PdvrqE3F0= -github.com/projectdiscovery/wappalyzergo v0.2.58/go.mod h1:lwuDLdAqWDZ1IL8OQnoNQ0t17UP9AQSvVuFcDAm4FpQ= +github.com/projectdiscovery/useragent v0.0.106 h1:9fS08MRUUJvfBskTxcXY9TA4X1TwpH6iJ3P3YNaXNlo= +github.com/projectdiscovery/useragent v0.0.106/go.mod h1:9oVMjgd7CchIsyeweyigIPtW83gpiGf2NtR6UM5XK+o= +github.com/projectdiscovery/utils v0.8.0 h1:8d79OCs5xGDNXdKxMUKMY/lgQSUWJMYB1B2Sx+oiqkQ= +github.com/projectdiscovery/utils v0.8.0/go.mod h1:CU6tjtyTRxBrnNek+GPJplw4IIHcXNZNKO09kWgqTdg= +github.com/projectdiscovery/wappalyzergo v0.2.63 h1:iSIU2rfPkHcpBSTol7S3PqgfBXn+JD56s4BsVEGxJ+o= +github.com/projectdiscovery/wappalyzergo v0.2.63/go.mod h1:8FtSVcmPRZU0g1euBpdSYEBHIvB7Zz9MOb754ZqZmfU= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/refraction-networking/utls v1.7.1 h1:dxg+jla3uocgN8HtX+ccwDr68uCBBO3qLrkZUbqkcw0= github.com/refraction-networking/utls v1.7.1/go.mod h1:TUhh27RHMGtQvjQq+RyO11P6ZNQNBb3N0v7wsEjKAIQ= @@ -382,6 +389,8 @@ github.com/sashabaranov/go-openai v1.37.0 h1:hQQowgYm4OXJ1Z/wTrE+XZaO20BYsL0R3uR github.com/sashabaranov/go-openai v1.37.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc= github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/seh-msft/burpxml v1.0.1 h1:5G3QPSzvfA1WcX7LkxmKBmK2RnNyGviGWnJPumE0nwg= +github.com/seh-msft/burpxml v1.0.1/go.mod h1:lTViCHPtGGS0scK0B4krm6Ld1kVZLWzQccwUomRc58I= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= @@ -393,14 +402,18 @@ github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnj github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg= -github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= +github.com/sorairolake/lzip-go v0.3.8 h1:j5Q2313INdTA80ureWYRhX+1K78mUXfMoPZCw/ivWik= +github.com/sorairolake/lzip-go v0.3.8/go.mod h1:JcBqGMV0frlxwrsE9sMWXDjqn3EeVf0/54YPsw66qkU= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -413,8 +426,6 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= -github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= @@ -449,8 +460,8 @@ github.com/vulncheck-oss/go-exploit v1.51.0 h1:HTmJ4Q94tbEDPb35mQZn6qMg4rT+Sw9n+ github.com/vulncheck-oss/go-exploit v1.51.0/go.mod h1:J28w0dLnA6DnCrnBm9Sbt6smX8lvztnnN2wCXy7No6c= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.2/go.mod h1:/hGscit36Yt+wammfBBwdMdxBT8btsTt6KvwO9OvMyM= -github.com/weppos/publicsuffix-go v0.50.1 h1:elrBHeSkS/eIb169+DnLrknqmdP4AjT0Q0tEdytz1Og= -github.com/weppos/publicsuffix-go v0.50.1/go.mod h1:znn0JVXjcR5hpUl9pbEogwH6I710rA1AX0QQPT0bf+k= +github.com/weppos/publicsuffix-go v0.50.2 h1:KsJFc8IEKTJovM46SRCnGNsM+rFShxcs6VEHjOJcXzE= +github.com/weppos/publicsuffix-go v0.50.2/go.mod h1:CbQCKDtXF8UcT7hrxeMa0MDjwhpOI9iYOU7cfq+yo8k= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= @@ -519,8 +530,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -553,8 +564,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= -golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= +golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -585,8 +596,8 @@ golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -608,8 +619,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -639,6 +650,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -650,8 +662,8 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -664,8 +676,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -681,8 +693,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= @@ -716,8 +728,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= +golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -763,8 +775,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= -gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= diff --git a/runner/cpe.go b/runner/cpe.go new file mode 100644 index 00000000..76444fa9 --- /dev/null +++ b/runner/cpe.go @@ -0,0 +1,225 @@ +package runner + +import ( + "encoding/json" + "fmt" + "strings" + + awesomesearchqueries "github.com/projectdiscovery/awesome-search-queries" +) + +type CPEInfo struct { + Product string `json:"product,omitempty"` + Vendor string `json:"vendor,omitempty"` + CPE string `json:"cpe,omitempty"` +} + +type CPEDetector struct { + titlePatterns map[string][]CPEInfo + bodyPatterns map[string][]CPEInfo + faviconPatterns map[string][]CPEInfo +} + +type rawQuery struct { + Name string `json:"name"` + Vendor json.RawMessage `json:"vendor"` + Type string `json:"type"` + Engines []rawEngine `json:"engines"` +} + +type rawEngine struct { + Platform string `json:"platform"` + Queries []string `json:"queries"` +} + +func NewCPEDetector() (*CPEDetector, error) { + data, err := awesomesearchqueries.GetQueries() + if err != nil { + return nil, fmt.Errorf("failed to load queries: %w", err) + } + + var queries []rawQuery + if err := json.Unmarshal(data, &queries); err != nil { + return nil, fmt.Errorf("failed to parse queries: %w", err) + } + + detector := &CPEDetector{ + titlePatterns: make(map[string][]CPEInfo), + bodyPatterns: make(map[string][]CPEInfo), + faviconPatterns: make(map[string][]CPEInfo), + } + + for _, q := range queries { + vendor := parseVendor(q.Vendor) + info := CPEInfo{ + Product: q.Name, + Vendor: vendor, + CPE: generateCPE(vendor, q.Name), + } + + for _, engine := range q.Engines { + for _, query := range engine.Queries { + detector.extractPattern(query, info) + } + } + } + + return detector, nil +} + +func parseVendor(raw json.RawMessage) string { + var vendorStr string + if err := json.Unmarshal(raw, &vendorStr); err == nil { + return vendorStr + } + + var vendorSlice []string + if err := json.Unmarshal(raw, &vendorSlice); err == nil && len(vendorSlice) > 0 { + return vendorSlice[0] + } + + return "" +} + +func generateCPE(vendor, product string) string { + if vendor == "" || product == "" { + return "" + } + return fmt.Sprintf("cpe:2.3:a:%s:%s:*:*:*:*:*:*:*:*", + strings.ToLower(strings.ReplaceAll(vendor, " ", "_")), + strings.ToLower(strings.ReplaceAll(product, " ", "_"))) +} + +func (d *CPEDetector) extractPattern(query string, info CPEInfo) { + query = strings.TrimSpace(query) + + titlePrefixes := []string{ + "http.title:", + "title=", + "title==", + "intitle:", + "title:", + "title='", + `title="`, + } + + for _, prefix := range titlePrefixes { + if strings.HasPrefix(strings.ToLower(query), strings.ToLower(prefix)) { + pattern := extractQuotedValue(strings.TrimPrefix(query, prefix)) + pattern = strings.TrimPrefix(pattern, prefix[:len(prefix)-1]) + if pattern != "" { + pattern = strings.ToLower(pattern) + d.titlePatterns[pattern] = appendUnique(d.titlePatterns[pattern], info) + } + return + } + } + + bodyPrefixes := []string{ + "http.html:", + "body=", + "body==", + "intext:", + } + + for _, prefix := range bodyPrefixes { + if strings.HasPrefix(strings.ToLower(query), strings.ToLower(prefix)) { + pattern := extractQuotedValue(strings.TrimPrefix(query, prefix)) + if pattern != "" { + pattern = strings.ToLower(pattern) + d.bodyPatterns[pattern] = appendUnique(d.bodyPatterns[pattern], info) + } + return + } + } + + faviconPrefixes := []string{ + "http.favicon.hash:", + "icon_hash=", + "icon_hash==", + } + + for _, prefix := range faviconPrefixes { + if strings.HasPrefix(strings.ToLower(query), strings.ToLower(prefix)) { + pattern := extractQuotedValue(strings.TrimPrefix(query, prefix)) + if pattern != "" { + d.faviconPatterns[pattern] = appendUnique(d.faviconPatterns[pattern], info) + } + return + } + } +} + +func extractQuotedValue(s string) string { + s = strings.TrimSpace(s) + + if len(s) >= 2 { + if (s[0] == '"' && s[len(s)-1] == '"') || (s[0] == '\'' && s[len(s)-1] == '\'') { + s = s[1 : len(s)-1] + } + } + + if idx := strings.Index(s, "\" ||"); idx > 0 { + s = s[:idx] + } + if idx := strings.Index(s, "' ||"); idx > 0 { + s = s[:idx] + } + + return strings.TrimSpace(s) +} + +func appendUnique(slice []CPEInfo, info CPEInfo) []CPEInfo { + for _, existing := range slice { + if existing.Product == info.Product && existing.Vendor == info.Vendor { + return slice + } + } + return append(slice, info) +} + +func (d *CPEDetector) Detect(title, body, faviconHash string) []CPEInfo { + seen := make(map[string]bool) + var results []CPEInfo + + titleLower := strings.ToLower(title) + bodyLower := strings.ToLower(body) + + for pattern, infos := range d.titlePatterns { + if strings.Contains(titleLower, pattern) { + for _, info := range infos { + key := info.Product + "|" + info.Vendor + if !seen[key] { + seen[key] = true + results = append(results, info) + } + } + } + } + + for pattern, infos := range d.bodyPatterns { + if strings.Contains(bodyLower, pattern) { + for _, info := range infos { + key := info.Product + "|" + info.Vendor + if !seen[key] { + seen[key] = true + results = append(results, info) + } + } + } + } + + if faviconHash != "" { + if infos, ok := d.faviconPatterns[faviconHash]; ok { + for _, info := range infos { + key := info.Product + "|" + info.Vendor + if !seen[key] { + seen[key] = true + results = append(results, info) + } + } + } + } + + return results +} diff --git a/runner/options.go b/runner/options.go index 0e41fd24..ce749600 100644 --- a/runner/options.go +++ b/runner/options.go @@ -2,7 +2,6 @@ package runner import ( "fmt" - "math" "os" "path/filepath" "regexp" @@ -23,7 +22,8 @@ import ( "github.com/projectdiscovery/httpx/common/customlist" customport "github.com/projectdiscovery/httpx/common/customports" fileutilz "github.com/projectdiscovery/httpx/common/fileutil" - "github.com/projectdiscovery/httpx/common/httpx" + httpxcommon "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/httpx/common/inputformats" "github.com/projectdiscovery/httpx/common/stringz" "github.com/projectdiscovery/networkpolicy" pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" @@ -86,6 +86,8 @@ type ScanOptions struct { NoFallback bool NoFallbackScheme bool TechDetect bool + CPEDetect bool + WordPress bool StoreChain bool StoreVisionReconClusters bool MaxResponseBodySizeToSave int @@ -149,6 +151,8 @@ func (s *ScanOptions) Clone() *ScanOptions { NoFallback: s.NoFallback, NoFallbackScheme: s.NoFallbackScheme, TechDetect: s.TechDetect, + CPEDetect: s.CPEDetect, + WordPress: s.WordPress, StoreChain: s.StoreChain, OutputExtractRegex: s.OutputExtractRegex, MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave, @@ -188,6 +192,7 @@ type Options struct { SocksProxy string Proxy string InputFile string + InputMode string InputTargetHost goflags.StringSlice Methods string RequestURI string @@ -257,6 +262,8 @@ type Options struct { NoFallback bool NoFallbackScheme bool TechDetect bool + CPEDetect bool + WordPress bool CustomFingerprintFile string TLSGrab bool protocol string @@ -346,6 +353,8 @@ type Options struct { // AssetFileUpload AssetFileUpload string TeamID string + // SecretFile is the path to the secret file for authentication + SecretFile string // OnClose adds a callback function that is invoked when httpx is closed // to be exact at end of existing closures OnClose func() @@ -370,6 +379,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.InputFile, "list", "l", "", "input file containing list of hosts to process"), flagSet.StringVarP(&options.InputRawRequest, "request", "rr", "", "file containing raw request"), flagSet.StringSliceVarP(&options.InputTargetHost, "target", "u", nil, "input target host(s) to probe", goflags.CommaSeparatedStringSliceOptions), + flagSet.StringVarP(&options.InputMode, "input-mode", "im", "", fmt.Sprintf("mode of input file (%s)", inputformats.SupportedFormats())), ) flagSet.CreateGroup("Probes", "Probes", @@ -388,6 +398,8 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"), flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"), flagSet.StringVarP(&options.CustomFingerprintFile, "custom-fingerprint-file", "cff", "", "path to a custom fingerprint file for technology detection"), + flagSet.BoolVar(&options.CPEDetect, "cpe", false, "display CPE (Common Platform Enumeration) based on awesome-search-queries"), + flagSet.BoolVarP(&options.WordPress, "wordpress", "wp", false, "display WordPress plugins and themes"), flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"), flagSet.BoolVarP(&options.OutputWebSocket, "websocket", "ws", false, "display server using websocket"), flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"), @@ -516,6 +528,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"), flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "Disable Stdin processing"), flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"), + flagSet.StringVarP(&options.SecretFile, "secret-file", "sf", "", "path to the secret file for authentication"), ) flagSet.CreateGroup("debug", "Debug", @@ -541,8 +554,8 @@ func ParseOptions() *Options { flagSet.IntVar(&options.Retries, "retries", 0, "number of retries"), flagSet.IntVar(&options.Timeout, "timeout", 10, "timeout in seconds"), flagSet.DurationVar(&options.Delay, "delay", -1, "duration between each http request (eg: 200ms, 1s)"), - flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "max response size to save in bytes"), - flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", math.MaxInt32, "max response size to read in bytes"), + flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", int(httpxcommon.DefaultMaxResponseBodySize), "max response size to save in bytes"), + flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", int(httpxcommon.DefaultMaxResponseBodySize), "max response size to read in bytes"), ) flagSet.CreateGroup("cloud", "Cloud", @@ -670,6 +683,17 @@ func (options *Options) ValidateOptions() error { return fmt.Errorf("file '%s' does not exist", options.InputRawRequest) } + if options.SecretFile != "" && !fileutil.FileExists(options.SecretFile) { + return fmt.Errorf("secret file '%s' does not exist", options.SecretFile) + } + if options.InputMode != "" && inputformats.GetFormat(options.InputMode) == nil { + return fmt.Errorf("invalid input mode '%s', supported formats: %s", options.InputMode, inputformats.SupportedFormats()) + } + + if options.InputMode != "" && options.InputFile == "" { + return errors.New("-im/-input-mode requires -l/-list to specify an input file") + } + if options.Silent { incompatibleFlagsList := flagsIncompatibleWithSilent(options) if len(incompatibleFlagsList) > 0 { @@ -772,7 +796,7 @@ func (options *Options) ValidateOptions() error { options.OutputCDN = "true" } - if !stringsutil.EqualFoldAny(options.Protocol, string(httpx.UNKNOWN), string(httpx.HTTP11)) { + if !stringsutil.EqualFoldAny(options.Protocol, string(httpxcommon.UNKNOWN), string(httpxcommon.HTTP11)) { return fmt.Errorf("invalid protocol: %s", options.Protocol) } diff --git a/runner/runner.go b/runner/runner.go index 2bf4dd44..14b91625 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -35,7 +35,9 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" + "github.com/projectdiscovery/httpx/common/inputformats" "github.com/projectdiscovery/httpx/common/pagetypeclassifier" + "github.com/projectdiscovery/httpx/common/authprovider" "github.com/projectdiscovery/httpx/static" "github.com/projectdiscovery/mapcidr/asn" "github.com/projectdiscovery/networkpolicy" @@ -81,6 +83,8 @@ type Runner struct { options *Options hp *httpx.HTTPX wappalyzer *wappalyzer.Wappalyze + cpeDetector *CPEDetector + wpDetector *WordPressDetector scanopts ScanOptions hm *hybrid.HybridMap excludeCdn bool @@ -92,6 +96,7 @@ type Runner struct { pHashClusters []pHashCluster simHashes gcache.Cache[uint64, struct{}] // Include simHashes for efficient duplicate detection httpApiEndpoint *Server + authProvider authprovider.AuthProvider } func (r *Runner) HTTPX() *httpx.HTTPX { @@ -133,9 +138,26 @@ func New(options *Options) (*Runner, error) { return nil, errors.Wrap(err, "could not create wappalyzer client") } + if options.CPEDetect || options.JSONOutput || options.CSVOutput { + runner.cpeDetector, err = NewCPEDetector() + if err != nil { + gologger.Warning().Msgf("Could not create CPE detector: %s", err) + } + } + + if options.WordPress || options.JSONOutput || options.CSVOutput { + runner.wpDetector, err = NewWordPressDetector() + if err != nil { + gologger.Warning().Msgf("Could not create WordPress detector: %s", err) + } + } + if options.StoreResponseDir != "" { - _ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt")) - _ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt")) + // Don't remove index files if skip-dedupe is enabled (we want to append, not truncate) + if !options.SkipDedupe { + _ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt")) + _ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt")) + } } httpxOptions := httpx.DefaultOptions @@ -297,6 +319,8 @@ func New(options *Options) (*Runner, error) { scanopts.NoFallback = options.NoFallback scanopts.NoFallbackScheme = options.NoFallbackScheme scanopts.TechDetect = options.TechDetect || options.JSONOutput || options.CSVOutput || options.AssetUpload + scanopts.CPEDetect = options.CPEDetect || options.JSONOutput || options.CSVOutput + scanopts.WordPress = options.WordPress || options.JSONOutput || options.CSVOutput scanopts.StoreChain = options.StoreChain scanopts.StoreVisionReconClusters = options.StoreVisionReconClusters scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave @@ -391,6 +415,16 @@ func New(options *Options) (*Runner, error) { } runner.pageTypeClassifier = pageTypeClassifier + if options.SecretFile != "" { + authProviderOpts := &authprovider.AuthProviderOptions{ + SecretsFiles: []string{options.SecretFile}, + } + runner.authProvider, err = authprovider.NewAuthProvider(authProviderOpts) + if err != nil { + return nil, errors.Wrap(err, "could not create auth provider") + } + } + if options.HttpApiEndpoint != "" { apiServer := NewServer(options.HttpApiEndpoint, options) gologger.Info().Msgf("Listening api endpoint on: %s", options.HttpApiEndpoint) @@ -481,27 +515,44 @@ func (r *Runner) prepareInputPaths() { } } +var duplicateTargetErr = errors.New("duplicate target") + func (r *Runner) prepareInput() { var numHosts int // check if input target host(s) have been provided if len(r.options.InputTargetHost) > 0 { for _, target := range r.options.InputTargetHost { - expandedTarget, _ := r.countTargetFromRawTarget(target) - if expandedTarget > 0 { + expandedTarget, err := r.countTargetFromRawTarget(target) + if err == nil && expandedTarget > 0 { numHosts += expandedTarget - r.hm.Set(target, nil) //nolint + r.hm.Set(target, []byte("1")) //nolint + } else if r.options.SkipDedupe && errors.Is(err, duplicateTargetErr) { + if v, ok := r.hm.Get(target); ok { + cnt, _ := strconv.Atoi(string(v)) + _ = r.hm.Set(target, []byte(strconv.Itoa(cnt+1))) + numHosts += 1 + } } } } // check if file has been provided if fileutil.FileExists(r.options.InputFile) { - finput, err := os.Open(r.options.InputFile) - if err != nil { - gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err) - } - numHosts, err = r.loadAndCloseFile(finput) - if err != nil { - gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err) + // check if input mode is specified for special format handling + if format := r.getInputFormat(); format != nil { + numTargets, err := r.loadFromFormat(r.options.InputFile, format) + if err != nil { + gologger.Fatal().Msgf("Could not parse input file '%s': %s\n", r.options.InputFile, err) + } + numHosts = numTargets + } else { + finput, err := os.Open(r.options.InputFile) + if err != nil { + gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err) + } + numHosts, err = r.loadAndCloseFile(finput) + if err != nil { + gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err) + } } } else if r.options.InputFile != "" { files, err := fileutilz.ListFilesWithPattern(r.options.InputFile) @@ -595,19 +646,52 @@ func (r *Runner) testAndSet(k string) bool { return true } +// getInputFormat returns the format for the configured input mode. +// Returns nil if no input mode is configured, or logs fatal if the format is invalid. +func (r *Runner) getInputFormat() inputformats.Format { + if r.options.InputMode == "" { + return nil + } + format := inputformats.GetFormat(r.options.InputMode) + if format == nil { + gologger.Fatal().Msgf("Invalid input mode '%s'. Supported: %s\n", r.options.InputMode, inputformats.SupportedFormats()) + } + return format +} + func (r *Runner) streamInput() (chan string, error) { out := make(chan string) go func() { defer close(out) if fileutil.FileExists(r.options.InputFile) { - fchan, err := fileutil.ReadFile(r.options.InputFile) - if err != nil { - return - } - for item := range fchan { - if r.options.SkipDedupe || r.testAndSet(item) { - out <- item + // check if input mode is specified for special format handling + if format := r.getInputFormat(); format != nil { + finput, err := os.Open(r.options.InputFile) + if err != nil { + gologger.Error().Msgf("Could not open input file '%s': %s\n", r.options.InputFile, err) + return + } + defer finput.Close() + if err := format.Parse(finput, func(item string) bool { + item = strings.TrimSpace(item) + if r.options.SkipDedupe || r.testAndSet(item) { + out <- item + } + return true + }); err != nil { + gologger.Error().Msgf("Could not parse input file '%s': %s\n", r.options.InputFile, err) + return + } + } else { + fchan, err := fileutil.ReadFile(r.options.InputFile) + if err != nil { + return + } + for item := range fchan { + if r.options.SkipDedupe || r.testAndSet(item) { + out <- item + } } } } else if r.options.InputFile != "" { @@ -647,22 +731,54 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { for scanner.Scan() { target := strings.TrimSpace(scanner.Text()) // Used just to get the exact number of targets - expandedTarget, _ := r.countTargetFromRawTarget(target) - if expandedTarget > 0 { + expandedTarget, err := r.countTargetFromRawTarget(target) + if err == nil && expandedTarget > 0 { numTargets += expandedTarget - r.hm.Set(target, nil) //nolint + r.hm.Set(target, []byte("1")) //nolint + } else if r.options.SkipDedupe && errors.Is(err, duplicateTargetErr) { + if v, ok := r.hm.Get(target); ok { + cnt, _ := strconv.Atoi(string(v)) + _ = r.hm.Set(target, []byte(strconv.Itoa(cnt+1))) + numTargets += 1 + } } } err = finput.Close() return numTargets, err } +func (r *Runner) loadFromFormat(filePath string, format inputformats.Format) (numTargets int, err error) { + finput, err := os.Open(filePath) + if err != nil { + return 0, err + } + defer finput.Close() + + err = format.Parse(finput, func(target string) bool { + target = strings.TrimSpace(target) + expandedTarget, countErr := r.countTargetFromRawTarget(target) + if countErr == nil && expandedTarget > 0 { + numTargets += expandedTarget + r.hm.Set(target, []byte("1")) //nolint + } else if r.options.SkipDedupe && errors.Is(countErr, duplicateTargetErr) { + if v, ok := r.hm.Get(target); ok { + cnt, _ := strconv.Atoi(string(v)) + _ = r.hm.Set(target, []byte(strconv.Itoa(cnt+1))) + numTargets += 1 + } + } + return true + }) + return numTargets, err +} + func (r *Runner) countTargetFromRawTarget(rawTarget string) (numTargets int, err error) { if rawTarget == "" { return 0, nil } + if _, ok := r.hm.Get(rawTarget); ok { - return 0, nil + return 0, duplicateTargetErr } expandedTarget := 0 @@ -893,7 +1009,8 @@ func (r *Runner) RunEnumeration() { gologger.Fatal().Msgf("Could not create response directory '%s': %s\n", responseDirPath, err) } indexPath := filepath.Join(responseDirPath, "index.txt") - if r.options.Resume { + // Append if resume is enabled or skip-dedupe is enabled (never truncate with -sd) + if r.options.Resume || r.options.SkipDedupe { indexFile, err = os.OpenFile(indexPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) } else { indexFile, err = os.Create(indexPath) @@ -907,7 +1024,8 @@ func (r *Runner) RunEnumeration() { if r.options.Screenshot { var err error indexScreenshotPath := filepath.Join(r.options.StoreResponseDir, "screenshot", "index_screenshot.txt") - if r.options.Resume { + // Append if resume is enabled or skip-dedupe is enabled (never truncate with -sd) + if r.options.Resume || r.options.SkipDedupe { indexScreenshotFile, err = os.OpenFile(indexScreenshotPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) } else { indexScreenshotFile, err = os.Create(indexScreenshotPath) @@ -1099,10 +1217,8 @@ func (r *Runner) RunEnumeration() { // store responses or chain in directory if resp.Err == nil { URL, _ := urlutil.Parse(resp.URL) - domainFile := resp.Method + ":" + URL.EscapedString() - hash := hashes.Sha1([]byte(domainFile)) - domainResponseFile := fmt.Sprintf("%s.txt", hash) - screenshotResponseFile := fmt.Sprintf("%s.png", hash) + domainResponseFile := fmt.Sprintf("%s.txt", resp.FileNameHash) + screenshotResponseFile := fmt.Sprintf("%s.png", resp.FileNameHash) hostFilename := strings.ReplaceAll(URL.Host, ":", "_") domainResponseBaseDir := filepath.Join(r.options.StoreResponseDir, "response") domainScreenshotBaseDir := filepath.Join(r.options.StoreResponseDir, "screenshot") @@ -1302,14 +1418,28 @@ func (r *Runner) RunEnumeration() { } } - if len(r.options.requestURIs) > 0 { - for _, p := range r.options.requestURIs { - scanopts := r.scanopts.Clone() - scanopts.RequestURI = p - r.process(k, wg, r.hp, protocol, scanopts, output) + runProcess := func(times int) { + for i := 0; i < times; i++ { + if len(r.options.requestURIs) > 0 { + for _, p := range r.options.requestURIs { + scanopts := r.scanopts.Clone() + scanopts.RequestURI = p + r.process(k, wg, r.hp, protocol, scanopts, output) + } + } else { + r.process(k, wg, r.hp, protocol, &r.scanopts, output) + } } - } else { - r.process(k, wg, r.hp, protocol, &r.scanopts, output) + } + + if r.options.Stream { + runProcess(1) + } else if v, ok := r.hm.Get(k); ok { + cnt, err := strconv.Atoi(string(v)) + if err != nil || cnt <= 0 { + cnt = 1 + } + runProcess(cnt) } return nil @@ -1562,7 +1692,7 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { results <- httpx.Target{Host: target} return } - ips, _, _, err := getDNSData(hp, URL.Host) + ips, _, _, err := getDNSData(hp, URL.Hostname()) if err != nil || len(ips) == 0 { results <- httpx.Target{Host: target} return @@ -1655,6 +1785,16 @@ retry: } hp.SetCustomHeaders(req, hp.CustomHeaders) + + // Apply auth strategies if auth provider is configured + if r.authProvider != nil { + if strategies := r.authProvider.LookupURLX(URL); len(strategies) > 0 { + for _, strategy := range strategies { + strategy.ApplyOnRR(req) + } + } + } + // We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size) if scanopts.RequestBody != "" { req.ContentLength = int64(len(scanopts.RequestBody)) @@ -2201,7 +2341,7 @@ retry: domainResponseBaseDir := filepath.Join(scanopts.StoreResponseDirectory, "response") responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename) - var responsePath string + var responsePath, fileNameHash string // store response if scanopts.StoreResponse || scanopts.StoreChain { if r.options.OmitBody { @@ -2222,9 +2362,33 @@ retry: data = append(data, []byte("\n\n\n")...) data = append(data, []byte(fullURL)...) _ = fileutil.CreateFolder(responseBaseDir) - writeErr := os.WriteFile(responsePath, data, 0644) - if writeErr != nil { - gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) + + basePath := strings.TrimSuffix(responsePath, ".txt") + var idx int + for idx = 0; ; idx++ { + targetPath := responsePath + if idx > 0 { + targetPath = fmt.Sprintf("%s_%d.txt", basePath, idx) + } + f, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) + if err == nil { + _, writeErr := f.Write(data) + _ = f.Close() + if writeErr != nil { + gologger.Error().Msgf("Could not write to '%s': %s", targetPath, writeErr) + } + break + } + if !os.IsExist(err) { + gologger.Error().Msgf("Failed to create file '%s': %s", targetPath, err) + break + } + } + + if idx == 0 { + fileNameHash = hash + } else { + fileNameHash = fmt.Sprintf("%s_%d", hash, idx) } } @@ -2311,6 +2475,47 @@ retry: } } + var cpeMatches []CPEInfo + if r.cpeDetector != nil { + cpeMatches = r.cpeDetector.Detect(title, string(resp.Data), faviconMMH3) + if len(cpeMatches) > 0 && r.options.CPEDetect { + for _, cpe := range cpeMatches { + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Cyan(cpe.CPE).String()) + } else { + builder.WriteString(cpe.CPE) + } + builder.WriteRune(']') + } + } + } + + var wpInfo *WordPressInfo + if r.wpDetector != nil { + wpInfo = r.wpDetector.Detect(string(resp.Data)) + if wpInfo.HasData() && r.options.WordPress { + if len(wpInfo.Plugins) > 0 { + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Green("wp-plugins:" + strings.Join(wpInfo.Plugins, ",")).String()) + } else { + builder.WriteString("wp-plugins:" + strings.Join(wpInfo.Plugins, ",")) + } + builder.WriteRune(']') + } + if len(wpInfo.Themes) > 0 { + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Green("wp-themes:" + strings.Join(wpInfo.Themes, ",")).String()) + } else { + builder.WriteString("wp-themes:" + strings.Join(wpInfo.Themes, ",")) + } + builder.WriteRune(']') + } + } + } + result := Result{ Timestamp: time.Now(), Request: request, @@ -2374,6 +2579,9 @@ retry: RequestRaw: requestDump, Response: resp, FaviconData: faviconData, + FileNameHash: fileNameHash, + CPE: cpeMatches, + WordPress: wpInfo, } if resp.BodyDomains != nil { result.Fqdns = resp.BodyDomains.Fqdns @@ -2493,6 +2701,10 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, } clone.SetURL(resolvedURL) + // Update Host header to match resolved URL host (important after redirects) + if resolvedURL.Host != "" && resolvedURL.Host != clone.Host { + clone.Host = resolvedURL.Host + } respFav, err := hp.Do(clone, httpx.UnsafeOptions{}) if err != nil || len(respFav.Data) == 0 { tries++ diff --git a/runner/runner_test.go b/runner/runner_test.go index 850566b8..10b8320b 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/pkg/errors" _ "github.com/projectdiscovery/fdmax/autofdmax" "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/mapcidr/asn" @@ -67,6 +68,36 @@ func TestRunner_probeall_targets(t *testing.T) { require.ElementsMatch(t, expected, got, "could not expected output") } +func TestRunner_probeall_targets_with_port(t *testing.T) { + options := &Options{ + ProbeAllIPS: true, + } + r, err := New(options) + require.Nil(t, err, "could not create httpx runner") + + inputWithPort := "http://one.one.one.one:8080" + inputWithoutPort := "one.one.one.one" + + gotWithPort := []httpx.Target{} + for target := range r.targets(r.hp, inputWithPort) { + gotWithPort = append(gotWithPort, target) + } + + gotWithoutPort := []httpx.Target{} + for target := range r.targets(r.hp, inputWithoutPort) { + gotWithoutPort = append(gotWithoutPort, target) + } + + require.True(t, len(gotWithPort) > 0, "probe-all-ips with port should return at least one target") + require.True(t, len(gotWithoutPort) > 0, "probe-all-ips without port should return at least one target") + require.Equal(t, len(gotWithPort), len(gotWithoutPort), "probe-all-ips should return same number of IPs with or without port") + + for _, target := range gotWithPort { + require.Equal(t, inputWithPort, target.Host, "Host should be preserved with port") + require.NotEmpty(t, target.CustomIP, "CustomIP should be populated") + } +} + func TestRunner_cidr_targets(t *testing.T) { options := &Options{} r, err := New(options) @@ -124,7 +155,9 @@ func TestRunner_asn_targets(t *testing.T) { } func TestRunner_countTargetFromRawTarget(t *testing.T) { - options := &Options{} + options := &Options{ + SkipDedupe: false, + } r, err := New(options) require.Nil(t, err, "could not create httpx runner") @@ -139,7 +172,7 @@ func TestRunner_countTargetFromRawTarget(t *testing.T) { err = r.hm.Set(input, nil) require.Nil(t, err, "could not set value to hm") got, err = r.countTargetFromRawTarget(input) - require.Nil(t, err, "could not count targets") + require.True(t, errors.Is(err, duplicateTargetErr), "expected duplicate target error") require.Equal(t, expected, got, "got wrong output") input = "173.0.84.0/24" @@ -227,10 +260,10 @@ func TestCreateNetworkpolicyInstance_AllowDenyFlags(t *testing.T) { runner := &Runner{} tests := []struct { - name string - allow []string - deny []string - testCases []struct { + name string + allow []string + deny []string + testCases []struct { ip string expected bool reason string diff --git a/runner/types.go b/runner/types.go index ab013a70..4cde28ba 100644 --- a/runner/types.go +++ b/runner/types.go @@ -102,6 +102,9 @@ type Result struct { Response *httpx.Response `json:"-" csv:"-" mapstructure:"-"` FaviconData []byte `json:"-" csv:"-" mapstructure:"-"` Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"-" mapstructure:"trace"` + FileNameHash string `json:"-" csv:"-" mapstructure:"-"` + CPE []CPEInfo `json:"cpe,omitempty" csv:"cpe" mapstructure:"cpe"` + WordPress *WordPressInfo `json:"wordpress,omitempty" csv:"wordpress" mapstructure:"wordpress"` } type Trace struct { diff --git a/runner/wordpress.go b/runner/wordpress.go new file mode 100644 index 00000000..20efc242 --- /dev/null +++ b/runner/wordpress.go @@ -0,0 +1,118 @@ +package runner + +import ( + "bufio" + "bytes" + "regexp" + "strings" + + awesomesearchqueries "github.com/projectdiscovery/awesome-search-queries" +) + +type WordPressInfo struct { + Plugins []string `json:"plugins,omitempty"` + Themes []string `json:"themes,omitempty"` +} + +type WordPressDetector struct { + knownPlugins map[string]struct{} + knownThemes map[string]struct{} + pluginRegex *regexp.Regexp + themeRegex *regexp.Regexp +} + +func NewWordPressDetector() (*WordPressDetector, error) { + detector := &WordPressDetector{ + knownPlugins: make(map[string]struct{}), + knownThemes: make(map[string]struct{}), + } + + var err error + + detector.pluginRegex, err = regexp.Compile(`/wp-content/plugins/([a-zA-Z0-9_-]+)/`) + if err != nil { + return nil, err + } + + detector.themeRegex, err = regexp.Compile(`/wp-content/themes/([a-zA-Z0-9_-]+)/`) + if err != nil { + return nil, err + } + + pluginsData, err := awesomesearchqueries.GetWordPressPlugins() + if err != nil { + return nil, err + } + if err := detector.loadList(pluginsData, detector.knownPlugins); err != nil { + return nil, err + } + + themesData, err := awesomesearchqueries.GetWordPressThemes() + if err != nil { + return nil, err + } + if err := detector.loadList(themesData, detector.knownThemes); err != nil { + return nil, err + } + + return detector, nil +} + +func (d *WordPressDetector) loadList(data []byte, target map[string]struct{}) error { + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line != "" { + target[line] = struct{}{} + } + } + return scanner.Err() +} + +func (d *WordPressDetector) Detect(body string) *WordPressInfo { + if body == "" { + return nil + } + + info := &WordPressInfo{} + seenPlugins := make(map[string]struct{}) + seenThemes := make(map[string]struct{}) + + if matches := d.pluginRegex.FindAllStringSubmatch(body, -1); len(matches) > 0 { + for _, match := range matches { + if len(match) > 1 { + plugin := match[1] + if _, seen := seenPlugins[plugin]; !seen { + if _, known := d.knownPlugins[plugin]; known { + info.Plugins = append(info.Plugins, plugin) + seenPlugins[plugin] = struct{}{} + } + } + } + } + } + + if matches := d.themeRegex.FindAllStringSubmatch(body, -1); len(matches) > 0 { + for _, match := range matches { + if len(match) > 1 { + theme := match[1] + if _, seen := seenThemes[theme]; !seen { + if _, known := d.knownThemes[theme]; known { + info.Themes = append(info.Themes, theme) + seenThemes[theme] = struct{}{} + } + } + } + } + } + + if len(info.Plugins) == 0 && len(info.Themes) == 0 { + return nil + } + + return info +} + +func (w *WordPressInfo) HasData() bool { + return w != nil && (len(w.Plugins) > 0 || len(w.Themes) > 0) +}