Skip to content

Commit 7b5785e

Browse files
acabarbayeacabarbaye
authored
✨ [hashing] Add helpers to simplify hashing (#736)
<!-- Copyright (C) 2020-2022 Arm Limited or its affiliates and Contributors. All rights reserved. SPDX-License-Identifier: Apache-2.0 --> ### Description - Add some helpers to ease the calculation of hashes ### Test Coverage <!-- Please put an `x` in the correct box e.g. `[x]` to indicate the testing coverage of this change. --> - [x] This change is covered by existing or additional automated tests. - [ ] Manual testing has been performed (and evidence provided) as automated testing was not feasible. - [ ] Additional tests are not required for this change (e.g. documentation update).
1 parent 19be1a1 commit 7b5785e

File tree

6 files changed

+73
-61
lines changed

6 files changed

+73
-61
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
steps:
4747
- uses: actions/setup-python@v5
4848
- name: Install tools
49-
run: pip install detect-secrets==1.0.3 && pip list
49+
run: pip install detect-secrets[gibberish]==1.5.0 && pip list
5050
- uses: actions/checkout@v4
5151
with:
5252
fetch-depth: 0

‎.secrets.baseline‎

Lines changed: 44 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": "1.0.3",
2+
"version": "1.5.0",
33
"plugins_used": [
44
{
55
"name": "ArtifactoryDetector"
@@ -20,6 +20,15 @@
2020
{
2121
"name": "CloudantDetector"
2222
},
23+
{
24+
"name": "DiscordBotTokenDetector"
25+
},
26+
{
27+
"name": "GitHubTokenDetector"
28+
},
29+
{
30+
"name": "GitLabTokenDetector"
31+
},
2332
{
2433
"name": "HexHighEntropyString",
2534
"limit": 3.0
@@ -30,6 +39,9 @@
3039
{
3140
"name": "IbmCosHmacDetector"
3241
},
42+
{
43+
"name": "IPPublicDetector"
44+
},
3345
{
3446
"name": "JwtTokenDetector"
3547
},
@@ -43,9 +55,18 @@
4355
{
4456
"name": "NpmDetector"
4557
},
58+
{
59+
"name": "OpenAIDetector"
60+
},
4661
{
4762
"name": "PrivateKeyDetector"
4863
},
64+
{
65+
"name": "PypiTokenDetector"
66+
},
67+
{
68+
"name": "SendGridDetector"
69+
},
4970
{
5071
"name": "SlackDetector"
5172
},
@@ -58,6 +79,9 @@
5879
{
5980
"name": "StripeDetector"
6081
},
82+
{
83+
"name": "TelegramBotTokenDetector"
84+
},
6185
{
6286
"name": "TwilioKeyDetector"
6387
}
@@ -66,20 +90,26 @@
6690
{
6791
"path": "detect_secrets.filters.allowlist.is_line_allowlisted"
6892
},
69-
{
70-
"path": "detect_secrets.filters.common.is_baseline_file",
71-
"filename": ".secrets.baseline"
72-
},
7393
{
7494
"path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
7595
"min_level": 2
7696
},
97+
{
98+
"path": "detect_secrets.filters.gibberish.should_exclude_secret",
99+
"limit": 3.7
100+
},
77101
{
78102
"path": "detect_secrets.filters.heuristic.is_indirect_reference"
79103
},
80104
{
81105
"path": "detect_secrets.filters.heuristic.is_likely_id_string"
82106
},
107+
{
108+
"path": "detect_secrets.filters.heuristic.is_lock_file"
109+
},
110+
{
111+
"path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
112+
},
83113
{
84114
"path": "detect_secrets.filters.heuristic.is_potential_uuid"
85115
},
@@ -89,68 +119,25 @@
89119
{
90120
"path": "detect_secrets.filters.heuristic.is_sequential_string"
91121
},
122+
{
123+
"path": "detect_secrets.filters.heuristic.is_swagger_file"
124+
},
92125
{
93126
"path": "detect_secrets.filters.heuristic.is_templated_secret"
94127
},
95128
{
96129
"path": "detect_secrets.filters.regex.should_exclude_file",
97130
"pattern": [
98-
".*go\\.sum$",
99131
".*\\.html$",
100132
".*\\.properties$",
101-
"ci.yml",
102-
"^workflows/.*",
103-
"^\\.git[\\\\/]"
133+
"^\\.git[\\\\/]",
134+
".*go\\.sum$",
135+
".*codegen.*",
136+
"workflows/.*"
104137
]
105138
}
106139
],
107140
"results": {
108-
"utils/config/fixtures/config-test.json": [
109-
{
110-
"type": "Secret Keyword",
111-
"filename": "utils/config/fixtures/config-test.json",
112-
"hashed_secret": "e38ad214943daad1d64c102faec29de4afe9da3d",
113-
"is_verified": false,
114-
"line_number": 10
115-
},
116-
{
117-
"type": "Secret Keyword",
118-
"filename": "utils/config/fixtures/config-test.json",
119-
"hashed_secret": "2aa60a8ff7fcd473d321e0146afd9e26df395147",
120-
"is_verified": false,
121-
"line_number": 19
122-
}
123-
],
124-
"utils/config/service_configuration_test.go": [
125-
{
126-
"type": "Secret Keyword",
127-
"filename": "utils/config/service_configuration_test.go",
128-
"hashed_secret": "ddcec2f503a5d58f432a0beee3fb9544fa581f54",
129-
"is_verified": false,
130-
"line_number": 37
131-
},
132-
{
133-
"type": "Secret Keyword",
134-
"filename": "utils/config/service_configuration_test.go",
135-
"hashed_secret": "7ca1cc114e7e5f955880bb96a5bf391b4dc20ab6",
136-
"is_verified": false,
137-
"line_number": 535
138-
},
139-
{
140-
"type": "Secret Keyword",
141-
"filename": "utils/config/service_configuration_test.go",
142-
"hashed_secret": "11519c144be4850d95b34220a40030cbd5a36b57",
143-
"is_verified": false,
144-
"line_number": 630
145-
},
146-
{
147-
"type": "Secret Keyword",
148-
"filename": "utils/config/service_configuration_test.go",
149-
"hashed_secret": "15fae91d8fa7f2c531c1cf3ddc745e1f4473c02d",
150-
"is_verified": false,
151-
"line_number": 637
152-
}
153-
],
154141
"utils/filesystem/filehash_test.go": [
155142
{
156143
"type": "Hex High Entropy String",
@@ -215,7 +202,7 @@
215202
"filename": "utils/hashing/hash_test.go",
216203
"hashed_secret": "30f0cbefb37316806a7024caee994baf8365fa53",
217204
"is_verified": false,
218-
"line_number": 132
205+
"line_number": 136
219206
}
220207
],
221208
"utils/sharedcache/common.go": [
@@ -276,5 +263,5 @@
276263
}
277264
]
278265
},
279-
"generated_at": "2025-09-26T12:33:39Z"
266+
"generated_at": "2025-10-27T11:47:07Z"
280267
}

‎DEVELOPMENT.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,6 @@ For the latter, False positive keys are stored in the [baseline](./.secrets.base
9494

9595
To flag individual false positives add comment `# pragma: allowlist secret` to line with secret
9696

97-
To add all suspected secrets in the repository (excluding ones with an allow secret comment), run `detect-secrets scan --all-files --exclude-files '.*go\.sum$' --exclude-files '.*\.html$' --exclude-files '.*\.properties$' --exclude-files 'ci.yml' --exclude-files '\.git' > .secrets.baseline`
97+
To add all suspected secrets in the repository (excluding ones with an allow secret comment), run `detect-secrets scan --all-files --exclude-files '.*\.html$' --exclude-files '.*\.properties$' --exclude-files '^\.git[\\/]' --exclude-files ".*go\.sum$" --exclude-files '.*codegen.*' --exclude-files 'workflows/.*' > .secrets.baseline`
9898

9999
If on Windows: then change the encoding of the .secrets.baseline file to UTF-8 then convert all `\` to `/` in the .secrets.baseline file

‎changes/20251027111917.feature‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
:sparkles: [hashing] Add helpers to simplify hashing

‎utils/hashing/hash.go‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
package hashing
77

88
import (
9+
"bytes"
910
"context"
1011
"crypto/md5" //nolint:gosec
1112
"crypto/sha1" //nolint:gosec
@@ -36,6 +37,10 @@ const (
3637
HashBlake2256 = "blake2b256" // https://www.blake2.net/
3738
)
3839

40+
var (
41+
SupportedHashingAlgorithms = []string{HashMd5, HashXXHash, HashSha1, HashSha256, HashMurmur, HashBlake2256}
42+
)
43+
3944
type hashingAlgo struct {
4045
Hash hash.Hash
4146
Type string
@@ -133,7 +138,7 @@ func NewHashingAlgorithm(htype string) (IHash, error) {
133138
}
134139

135140
if hash == nil {
136-
return nil, commonerrors.New(commonerrors.ErrNotFound, "could not find the corresponding hashing algorithm")
141+
return nil, commonerrors.Newf(commonerrors.ErrNotFound, "could not find the corresponding hashing algorithm. only %v are supported", SupportedHashingAlgorithms)
137142
}
138143
return newHashingAlgorithm(htype, hash)
139144
}
@@ -196,6 +201,21 @@ func CalculateHashWithContext(ctx context.Context, text, htype string) string {
196201
return CalculateStringHashWithContext(ctx, hashing, text)
197202
}
198203

204+
// CalculateHashFromReader returns the hash of element coming from a reader.
205+
func CalculateHashFromReader(ctx context.Context, htype string, reader io.Reader) (hash string, err error) {
206+
hashing, err := NewHashingAlgorithm(htype)
207+
if err != nil {
208+
return
209+
}
210+
hash, err = hashing.CalculateWithContext(ctx, reader)
211+
return
212+
}
213+
214+
// CalculateBytesHash returns the hash of a byte array
215+
func CalculateBytesHash(ctx context.Context, htype string, array []byte) (string, error) {
216+
return CalculateHashFromReader(ctx, htype, bytes.NewReader(array))
217+
}
218+
199219
// CalculateHashOfListOfStrings calculates the hash of some text using the requested htype hashing algorithm.
200220
func CalculateHashOfListOfStrings(ctx context.Context, htype string, text ...string) string {
201221
hashing, err := NewHashingAlgorithm(htype)

‎utils/hashing/hash_test.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,13 @@ func TestMd5(t *testing.T) {
6464
}, {
6565
Input: "CMSIS",
6666
Hash: "c61d595888f85f6d30e99ef6cacfcb7d",
67-
}}
67+
},
68+
}
6869
for _, testCase := range testCases {
6970
assert.Equal(t, testCase.Hash, CalculateMD5Hash(testCase.Input))
71+
hash, err := CalculateBytesHash(context.Background(), HashMd5, []byte(testCase.Input))
72+
require.NoError(t, err)
73+
assert.Equal(t, testCase.Hash, hash)
7074
}
7175
}
7276

0 commit comments

Comments
 (0)