Merge pull request #2 from secureCodeBox/hooks

Hooks Concept Implementation

Author: rfelber

.github/workflows/ci.yaml

@@ -10,7 +10,7 @@ jobs:
         run: ls
       - name: "Install npm dependencies in all parser sub projects"
         run: |
-          for dir in integrations/*/parser/
+          for dir in scanner/*/parser/
           do
               cd $dir
               if [ -f package.json ] && [ -f package-lock.json ]; then
@@ -20,9 +20,9 @@ jobs:
               # cd back
               cd -
           done
-      - name: "Install npm dependencies in all persistence sub projects"
+      - name: "Install npm dependencies in all hook sub projects"
         run: |
-          for dir in persistence/*/
+          for dir in hooks/*/
           do
               cd $dir
               if [ -f package.json ] && [ -f package-lock.json ]; then
@@ -35,7 +35,10 @@ jobs:
       - name: "Install npm test dependencies"
         run: |
           npm ci
-          cd integrations/
+          cd scanner/
+          npm ci
+          cd -
+          cd hooks/
           npm ci
       - name: "Run tests"
         run: |
@@ -106,7 +109,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: scbexperimental/parser-amass
-          path: ./integrations/amass/parser/
+          path: ./scanner/amass/parser/
           tag_with_ref: true
           tag_with_sha: true
       - uses: docker/build-push-action@v1
@@ -115,7 +118,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: scbexperimental/parser-kube-hunter
-          path: ./integrations/kube-hunter/parser/
+          path: ./scanner/kube-hunter/parser/
           tag_with_ref: true
           tag_with_sha: true
       - uses: docker/build-push-action@v1
@@ -124,7 +127,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: scbexperimental/parser-nikto
-          path: ./integrations/nikto/parser/
+          path: ./scanner/nikto/parser/
           tag_with_ref: true
           tag_with_sha: true
       - uses: docker/build-push-action@v1
@@ -133,7 +136,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: scbexperimental/parser-nmap
-          path: ./integrations/nmap/parser/
+          path: ./scanner/nmap/parser/
           tag_with_ref: true
           tag_with_sha: true
       - uses: docker/build-push-action@v1
@@ -142,7 +145,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: scbexperimental/parser-ssh-scan
-          path: ./integrations/ssh_scan/parser/
+          path: ./scanner/ssh_scan/parser/
           tag_with_ref: true
           tag_with_sha: true
       - uses: docker/build-push-action@v1
@@ -151,7 +154,16 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: scbexperimental/parser-sslyze
-          path: ./integrations/sslyze/parser/
+          path: ./scanner/sslyze/parser/
+          tag_with_ref: true
+          tag_with_sha: true
+      - uses: docker/build-push-action@v1
+        name: "Build & Push test-scan Parser Image"
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: scbexperimental/parser-test-scan
+          path: ./scanner/test-scan/parser/
           tag_with_ref: true
           tag_with_sha: true
       - uses: docker/build-push-action@v1
@@ -160,7 +172,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: scbexperimental/parser-trivy
-          path: ./integrations/trivy/parser/
+          path: ./scanner/trivy/parser/
           tag_with_ref: true
           tag_with_sha: true
       - uses: docker/build-push-action@v1
@@ -169,40 +181,65 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: scbexperimental/parser-zap
-          path: ./integrations/zap/parser/
+          path: ./scanner/zap/parser/
           tag_with_ref: true
           tag_with_sha: true
-  persistenceImages:
-    name: "Build / PersistenceProviders"
+  hookImages:
+    name: "Build / Hooks"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@master
       # SDK
       - uses: docker/build-push-action@v1
-        name: "Build & Push Persistence SDK"
+        name: "Build & Push Hook SDK"
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: scbexperimental/persistence-sdk-nodejs
-          path: ./persistence-sdk/nodejs/
+          repository: scbexperimental/hook-sdk-nodejs
+          path: ./hook-sdk/nodejs/
           tag_with_ref: true
       # Actual PersistenceProviders
       - uses: docker/build-push-action@v1
-        name: "Build & Push Elastic PersistenceProvider Image"
+        name: "Build & Push Elastic PersistenceProvider Hook Image"
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: scbexperimental/persistence-elastic
-          path: ./persistence/persistence-elastic/
+          path: ./hooks/persistence-elastic/
           tag_with_ref: true
       - uses: docker/build-push-action@v1
         name: "Build & Push Elastic PersistenceProvider Dashboard Importer Image"
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: scbexperimental/persistence-elastic-dashboard-importer
-          path: ./persistence/persistence-elastic/dashboardImporter/
+          path: ./hooks/persistence-elastic/dashboardImporter/
+          tag_with_ref: true
+      - uses: docker/build-push-action@v1
+        name: "Build & Push GenericWebhook Hook Image"
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: scbexperimental/generic-webhook
+          path: ./hooks/generic-webhook/
+          tag_with_ref: true
+      - uses: docker/build-push-action@v1
+        name: "Build & Push ImperativeSubsequentScans Hook Image"
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: scbexperimental/hook-imperative-subsequent-scans
+          path: ./hooks/imperative-subsequent-scans/
           tag_with_ref: true
+      - uses: docker/build-push-action@v1
+        name: "Build & Push UpdateField Hook Image"
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: scbexperimental/update-field
+          path: ./hooks/update-field/
+          tag_with_ref: true
+          tag_with_sha: true
   scannerImages:
     # Note we only build images for scanner that don't provider official public container images
     name: "Build / Scanner"
@@ -215,7 +252,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: scbexperimental/nmap
-          path: ./integrations/nmap/scanner/
+          path: ./scanner/nmap/scanner/
           # Note: not prefixed with a "v" as this seems to match nmap versioning standards
           tags: "7.80,7.80-1,latest"
       - uses: docker/build-push-action@v1
@@ -224,14 +261,23 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: scbexperimental/kube-hunter
-          path: ./integrations/kube-hunter/scanner/
+          path: ./scanner/kube-hunter/scanner/
           # Note: not prefixed with a "v" as this matches the aquasec/kube-hunter tags
           tags: "0.3.0,latest"
+      - uses: docker/build-push-action@v1
+        name: "Build & Push test-scan Scanner Image"
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: scbexperimental/test-scan
+          path: ./scanner/test-scan/scanner/
+          # Note: not prefixed with a "v" as this seems to match nmap versioning standards
+          tags: "latest"
   integrationTests:
     name: "Test / Integration / k8s ${{ matrix.k8sVersion }}"
     needs:
       - scannerImages
-      - persistenceImages
+      - hookImages
       - parserImages
       - operator
       - lurcher
@@ -276,23 +322,37 @@ jobs:
           cd tests/integration/ 
           npm ci
       # This steps should include Integration tests which are not related to a Specific Scanner
-      - name: "Generic Integration Tests"
+      - name: "Throws NoScanDefiniton Error Integration Tests"
         run: |
           cd tests/integration/
           npx jest --ci --color no-scan-definition-error
+      - name: "Hooks Integration Tests"
+        run: |
+          helm -n integration-tests install update-category ./hooks/update-field/ \
+            --set="image.tag=sha-$(git rev-parse --short HEAD)" \
+            --set="attribute.name=category" \
+            --set="attribute.value=fancy-category"
+          helm -n integration-tests install update-severity ./hooks/update-field/ \
+            --set="image.tag=sha-$(git rev-parse --short HEAD)" \
+            --set="attribute.name=severity" \
+            --set="attribute.value=high"
+          helm -n integration-tests install test-scan ./scanner/test-scan/ --set="parserImage.tag=sha-$(git rev-parse --short HEAD)"
+          cd tests/integration/
+          npx jest --ci --color read-write-hook
+          helm -n integration-tests uninstall test-scan update-category update-severity
       - name: "nmap Integration Tests"
         run: |
-          helm -n integration-tests install nmap ./integrations/nmap/ --set="parserImage.tag=sha-$(git rev-parse --short HEAD)"
+          helm -n integration-tests install nmap ./scanner/nmap/ --set="parserImage.tag=sha-$(git rev-parse --short HEAD)"
           cd tests/integration/
           npx jest --ci --color nmap
       - name: "kube-hunter Integration Tests"
         run: |
-          helm -n integration-tests install kube-hunter ./integrations/kube-hunter/ --set="parserImage.tag=sha-$(git rev-parse --short HEAD)"
+          helm -n integration-tests install kube-hunter ./scanner/kube-hunter/ --set="parserImage.tag=sha-$(git rev-parse --short HEAD)"
           cd tests/integration/
           npx jest --ci --color kube-hunter
       - name: "ssh-scan Integration Tests"
         run: |
-          helm -n integration-tests install ssh-scan ./integrations/ssh_scan/ --set="parserImage.tag=sha-$(git rev-parse --short HEAD)"
+          helm -n integration-tests install ssh-scan ./scanner/ssh_scan/ --set="parserImage.tag=sha-$(git rev-parse --short HEAD)"
           # Install dummy-ssh app
           helm -n demo-apps install dummy-ssh ./demo-apps/dummy-ssh/ --wait
           cd tests/integration/
@@ -304,6 +364,7 @@ jobs:
           kubectl -n integration-tests get jobs -o wide
           kubectl -n integration-tests get pods -o wide
       - name: "Inspect Operator"
+        if: failure()
         run: |
           echo "Deployment in namespace 'securecodebox-system'"
           kubectl -n securecodebox-system get deployments

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <img alt="secureCodeBox Logo" src="securecodebox-logo.svg" width="800px">
+  <img alt="secureCodeBox Logo" src="./docs/resources/securecodebox-logo.svg" width="800px">
 </p>
 
 <p align="center">
@@ -20,7 +20,9 @@
 - [Purpose of this Project](#purpose-of-this-project)
 - [Quickstart](#quickstart)
   - [Prerequisites](#prerequisites)
-  - [Deployment](#deployment)
+  - [Deployment (based on Helm)](#deployment-based-on-helm)
+  - [Examples](#examples)
+  - [Access Services](#access-services)
 - [How does it work?](#how-does-it-work)
 - [Architecture](#architecture)
 - [License](#license)
@@ -56,38 +58,80 @@ There is a german article about [Security DevOps – Angreifern (immer) einen Sc
 
 - kubernetes (last 4 mayor releases supported: `1.15`, `1.16`, `1.17` & `1.18`)
 
-### Deployment
+### Deployment (based on Helm)
 
 ```bash
-# Deploy secureCodeBox Operator
+# Deploy the secureCodeBox Operator first
 kubectl create namespace securecodebox-system
 helm -n securecodebox-system install securecodebox-operator ./operator/
 
-# Elasticsearch Persistence Provider Deployment
-helm install persistence-elastic ./persistence/persistence-elastic/
+# Deploy SCB scanner Charts for each security scanner you want to use (all optional)
+helm upgrade --install amass ./integrations/amass/
+helm upgrade --install kube-hunter ./integrations/kube-hunter/
+helm upgrade --install nikto ./integrations/nikto
+helm upgrade --install nmap ./integrations/nmap/
+helm upgrade --install ssh-scan ./integrations/ssh_scan/
+helm upgrade --install sslyze ./integrations/sslyze/
+helm upgrade --install trivy ./integrations/trivy/
+helm upgrade --install zap ./integrations/zap/
+helm upgrade --install wpscan ./integrations/wpscan/
+
+# Optional Deploy some Demo Apps for scanning
+helm upgrade --install dummy-ssh ./demo-apps/dummy-ssh/
+
+# Deploy secureCodeBox Hooks 
+helm upgrade --install aah ./hooks/add-attributes/
+helm upgrade --install gwh ./hooks/generic-webhook/
+helm upgrade --install issh ./hooks/imperative-subsequent-scans/
+
+## Persistence Provider: Elasticsearch
+helm upgrade --install elkh ./hooks/persistence-elastic/
+```
 
-# Deploy definitions for the integrated scanners
-helm install amass ./integrations/amass/
-helm install kube-hunter ./integrations/kube-hunter/
-helm install nikto ./integrations/nikto
-helm install nmap ./integrations/nmap/
-helm install ssh-scan ./integrations/ssh_scan/
-helm install sslyze ./integrations/sslyze/
-helm install zap ./integrations/zap/
+### Examples
 
+```bash
 # Now everything is installed. You can try deploying scans from the `operator/config/samples/` directory
+## Local Scan Examples
 
+### E.g. localhost nmap scan
+kubectl apply -f operator/config/samples/execution_v1_scan/nmap_localhost.yaml
+kubectl apply -f operator/config/samples/execution_v1_scan/kube-hunter_in_cluster.yaml
+
+## Public Scan Examples
 # E.g. www.securecodebox.io sslyze scan
+kubectl apply -f operator/config/samples/execution_v1_scan/nmap_securecodebox_io.yaml
+kubectl apply -f operator/config/samples/execution_v1_scan/amass_securecodebox_io.yaml
+kubectl apply -f operator/config/samples/execution_v1_scan/sslyze_securecodebox_io.yaml
+kubectl apply -f operator/config/samples/execution_v1_scan/nikto_securecodebox_io.yaml
+kubectl apply -f operator/config/samples/execution_v1_scan/ssh_iteratec_de.yaml
+kubectl apply -f operator/config/samples/execution_v1_scan/wpscan_nurdemteam_org.yaml
 kubectl apply -f operator/config/samples/execution_v1_scan/sslyze_securecodebox_io.yaml
+kubectl apply -f operator/config/samples/execution_v1_scan/trivy_mediawiki.yaml
+kubectl apply -f operator/config/samples/execution_v1_scan/trivy_juiceshop.yaml
+
 # Then get the current State of the Scan by running:
 kubectl get scans
 ```
 
+### Access Services
+
+* Minio UI:
+  * AccessKey: `kubectl get secret securecodebox-operator-minio -n securecodebox-system -o=jsonpath='{.data.accesskey}' | base64 --decode; echo`
+  * SecretKey: `kubectl get secret securecodebox-operator-minio -n securecodebox-system -o=jsonpath='{.data.secretkey}' | base64 --decode; echo`
+  * Port Forward Minio UI: `kubectl port-forward -n securecodebox-system service/securecodebox-operator-minio 9000:9000`
+* Elastic / Kibana UI:
+ * User: `elastic`
+ * Password: `kubectl get secret scb-elasticsearch-es-elastic-user -n scb-analytics -o=jsonpath='{.data.elastic}' | base64 --decode; echo`
+ * Port Forward Kibana: `kubectl port-forward -n default service/persistence-elastic-kibana 5601:5601`
+ * Port Forward Elasticsearch: `kubectl port-forward -n default service/elasticsearch-master 9200:9200` 
+
+
 ## How does it work?
 
 ## Architecture
 
-![secureCodeBox Architecture](scb-architecture.svg)
+![secureCodeBox Architecture](./docs/resources/scb-architecture.svg)
 
 ## License
 

docs/.gitkeep

@@ -0,0 +1 @@
+#

docs/_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-minimal