Merge pull request #240 from cerberauth/discover-well-known-paths

Discover Well-Known paths and leaked files

Author: emmanuelgautier

.github/workflows/scans.yml

@@ -45,7 +45,7 @@ jobs:
       - name: VulnAPI
         id: vulnapi
         run: |
-          go run main.go discover api http://localhost:8080 --sqa-opt-out
+          go run main.go discover api http://localhost:8080 --rate-limit 500 --sqa-opt-out
 
       - name: Stop Server
         if: ${{ always() }}

README.md

@@ -32,7 +32,7 @@ The Vulnerability Scanner CLI offers two methods for scanning APIs:
 
 ### Discover Command
 
-To discover target API useful information, execute the following command:
+To discover target API useful information, leaked files and well-known path execute the following command:
 
 ```bash
 vulnapi discover api [API_URL]
@@ -41,11 +41,12 @@ vulnapi discover api [API_URL]
 Example output:
 
 ```bash
-| WELL-KNOWN PATHS |                URL                 |
-|------------------|------------------------------------|
-| OpenAPI          | http://localhost:5000/openapi.json |
-| GraphQL          | N/A                                |
-
+|     TYPE      |                     URL                     |
+|---------------|---------------------------------------------|
+| OpenAPI       | http://localhost:5000/openapi.json          |
+| GraphQL       | http://localhost:5000/graphql               |
+| Well-Known    | http://localhost:8080/.well-known/jwks.json |
+| Exposed Files | http://localhost:8080/.env.dev              |
 
 | TECHNOLOGIE/SERVICE |     VALUE     |
 |---------------------|---------------|

internal/cmd/printtable/wellknown_paths_table.go

@@ -4,47 +4,57 @@ import (
 	"fmt"
 
 	"github.com/cerberauth/vulnapi/report"
+	"github.com/cerberauth/vulnapi/scan/discover"
 	discoverablegraphql "github.com/cerberauth/vulnapi/scan/discover/discoverable_graphql"
 	discoverableopenapi "github.com/cerberauth/vulnapi/scan/discover/discoverable_openapi"
+	exposedfiles "github.com/cerberauth/vulnapi/scan/discover/exposed_files"
+	wellknown "github.com/cerberauth/vulnapi/scan/discover/well-known"
 	"github.com/olekukonko/tablewriter"
 )
 
+func wellKnownPathsFromReport(r *report.ScanReport, header string) [][]string {
+	rows := [][]string{}
+	if r == nil || !r.HasData() {
+		return rows
+	}
+
+	data, ok := r.Data.(discover.DiscoverData)
+	if ok && len(data) > 0 {
+		rows = append(rows, []string{header, data[0].URL})
+	}
+
+	return rows
+}
+
 func WellKnownPathsScanReport(reporter *report.Reporter) {
-	openapiURL := ""
+	rows := [][]string{}
+
 	openapiReport := reporter.GetScanReportByID(discoverableopenapi.DiscoverableOpenAPIScanID)
-	if openapiReport != nil && openapiReport.HasData() {
-		openapiData, ok := openapiReport.Data.(discoverableopenapi.DiscoverableOpenAPIData)
-		if ok {
-			openapiURL = openapiData.URL
-		}
-	}
+	rows = append(rows, wellKnownPathsFromReport(openapiReport, "OpenAPI")...)
 
-	graphqlURL := ""
 	graphqlReport := reporter.GetScanReportByID(discoverablegraphql.DiscoverableGraphQLPathScanID)
-	if graphqlReport != nil && graphqlReport.HasData() {
-		graphqlData, ok := graphqlReport.Data.(discoverablegraphql.DiscoverableGraphQLPathData)
-		if ok {
-			graphqlURL = graphqlData.URL
-		}
-	}
+	rows = append(rows, wellKnownPathsFromReport(graphqlReport, "GraphQL")...)
 
-	if openapiURL == "" && graphqlURL == "" {
+	wellKnownReport := reporter.GetScanReportByID(wellknown.DiscoverableWellKnownScanID)
+	rows = append(rows, wellKnownPathsFromReport(wellKnownReport, "Well-Known")...)
+
+	exposedFiles := reporter.GetScanReportByID(exposedfiles.DiscoverableFilesScanID)
+	rows = append(rows, wellKnownPathsFromReport(exposedFiles, "Exposed Files")...)
+
+	if len(rows) == 0 {
 		return
 	}
 
 	fmt.Println()
-	headers := []string{"Well-Known Paths", "URL"}
+	headers := []string{"Type", "URL"}
 	table := CreateTable(headers)
 
 	tableColors := make([]tablewriter.Colors, len(headers))
 	tableColors[0] = tablewriter.Colors{tablewriter.Bold}
 	tableColors[1] = tablewriter.Colors{tablewriter.Bold}
 
-	if openapiURL != "" {
-		table.Rich([]string{"OpenAPI", openapiURL}, tableColors)
-	}
-	if graphqlURL != "" {
-		table.Rich([]string{"GraphQL", graphqlURL}, tableColors)
+	for _, row := range rows {
+		table.Rich(row, tableColors)
 	}
 
 	table.Render()

scan/broken_authentication/jwt/weak_secret/weak_secret.go

@@ -49,7 +49,8 @@ func ShouldBeScanned(securityScheme *auth.SecurityScheme) bool {
 
 var defaultJwtSecretDictionary = []string{"secret", "password", "123456", "changeme", "admin", "token"}
 
-const jwtSecretDictionarySeclistUrl = "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Passwords/scraped-JWT-secrets.txt"
+// From https://raw.githubusercontent.com/danielmiessler/SecLists/master/Passwords/scraped-JWT-secrets.txt
+const jwtSecretDictionarySeclistUrl = "https://raw.githubusercontent.com/cerberauth/vulnapi/main/seclist/lists/jwt-secrets.txt"
 
 func ScanHandler(op *operation.Operation, securityScheme *auth.SecurityScheme) (*report.ScanReport, error) {
 	vulnReport := report.NewIssueReport(issue).WithOperation(op).WithSecurityScheme(securityScheme)

scan/discover/discoverable_graphql/discoverable_graphql.go

@@ -29,20 +29,10 @@ var issue = report.Issue{
 	},
 }
 
-var graphqlSeclistUrl = "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/graphql.txt"
-var potentialGraphQLEndpoints = []string{
-	"/graphql",
-	"/graph",
-	"/api/graphql",
-	"/graphql/console",
-	"/v1/graphql",
-	"/v1/graphiql",
-}
+var graphqlSeclistUrl = "https://raw.githubusercontent.com/cerberauth/vulnapi/main/seclist/lists/graphql.txt"
 
 func ScanHandler(op *operation.Operation, securityScheme *auth.SecurityScheme) (*report.ScanReport, error) {
 	vulnReport := report.NewIssueReport(issue).WithOperation(op).WithSecurityScheme(securityScheme)
 	r := report.NewScanReport(DiscoverableGraphQLPathScanID, DiscoverableGraphQLPathScanName, op)
-	handler := discover.CreateURLScanHandler("GraphQL", graphqlSeclistUrl, potentialGraphQLEndpoints, r, vulnReport)
-
-	return handler(op, securityScheme)
+	return discover.DownloadAndScanURLs("GraphQL", graphqlSeclistUrl, r, vulnReport, op, securityScheme)
 }

scan/discover/discoverable_openapi/discoverable_openapi.go

@@ -29,19 +29,10 @@ var issue = report.Issue{
 	},
 }
 
-var openapiSeclistUrl = "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/swagger.txt"
-var potentialOpenAPIPaths = []string{
-	"/openapi",
-	"/api-docs.json",
-	"/api-docs.yaml",
-	"/api-docs.yml",
-	"/.well-known/openapi.yml",
-}
+var openapiSeclistUrl = "https://raw.githubusercontent.com/cerberauth/vulnapi/main/seclist/lists/swagger.txt"
 
 func ScanHandler(op *operation.Operation, securityScheme *auth.SecurityScheme) (*report.ScanReport, error) {
 	vulnReport := report.NewIssueReport(issue).WithOperation(op).WithSecurityScheme(securityScheme)
 	r := report.NewScanReport(DiscoverableOpenAPIScanID, DiscoverableOpenAPIScanName, op)
-	handler := discover.CreateURLScanHandler("OpenAPI", openapiSeclistUrl, potentialOpenAPIPaths, r, vulnReport)
-
-	return handler(op, securityScheme)
+	return discover.DownloadAndScanURLs("OpenAPI", openapiSeclistUrl, r, vulnReport, op, securityScheme)
 }

scan/discover/exposed_files/exposed_files.go

@@ -0,0 +1,38 @@
+package exposedfiles
+
+import (
+	"github.com/cerberauth/vulnapi/internal/auth"
+	"github.com/cerberauth/vulnapi/internal/operation"
+	"github.com/cerberauth/vulnapi/report"
+	"github.com/cerberauth/vulnapi/scan/discover"
+)
+
+const (
+	DiscoverableFilesScanID   = "discover.exposed_files"
+	DiscoverableFilesScanName = "Discoverable exposed files"
+)
+
+type DiscoverableFilesData = discover.DiscoverData
+
+var issue = report.Issue{
+	ID:   "discover.exposed_files",
+	Name: "Discoverable exposed files",
+
+	Classifications: &report.Classifications{
+		OWASP: report.OWASP_2023_SSRF,
+	},
+
+	CVSS: report.CVSS{
+		Version: 4.0,
+		Vector:  "CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:N/VI:N/VA:N/SC:N/SI:N/SA:N",
+		Score:   0,
+	},
+}
+
+var discoverableFilesSeclistUrl = "https://raw.githubusercontent.com/cerberauth/vulnapi/main/seclist/lists/exposed-paths.txt"
+
+func ScanHandler(op *operation.Operation, securityScheme *auth.SecurityScheme) (*report.ScanReport, error) {
+	vulnReport := report.NewIssueReport(issue).WithOperation(op).WithSecurityScheme(securityScheme)
+	r := report.NewScanReport(DiscoverableFilesScanID, DiscoverableFilesScanName, op)
+	return discover.DownloadAndScanURLs("Exposed Files", discoverableFilesSeclistUrl, r, vulnReport, op, securityScheme)
+}

scan/discover/exposed_files/exposed_files_test.go

@@ -0,0 +1,50 @@
+package exposedfiles_test
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/cerberauth/vulnapi/internal/auth"
+	"github.com/cerberauth/vulnapi/internal/operation"
+	"github.com/cerberauth/vulnapi/internal/request"
+	exposedfiles "github.com/cerberauth/vulnapi/scan/discover/exposed_files"
+	"github.com/jarcoal/httpmock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDiscoverableScanner_Passed_WhenNoDiscoverableGraphqlPathFound(t *testing.T) {
+	client := request.NewClient(request.NewClientOptions{
+		RateLimit: 500,
+	})
+	httpmock.ActivateNonDefault(client.Client)
+	defer httpmock.DeactivateAndReset()
+
+	op := operation.MustNewOperation(http.MethodGet, "http://localhost:8080/", nil, client)
+	httpmock.RegisterResponder(op.Method, op.URL.String(), httpmock.NewBytesResponder(http.StatusNoContent, nil))
+	httpmock.RegisterNoResponder(httpmock.NewBytesResponder(http.StatusNotFound, nil))
+
+	report, err := exposedfiles.ScanHandler(op, auth.MustNewNoAuthSecurityScheme())
+
+	require.NoError(t, err)
+	assert.Greater(t, httpmock.GetTotalCallCount(), 7)
+	assert.True(t, report.Issues[0].HasPassed())
+}
+
+func TestDiscoverableScanner_Failed_WhenOneGraphQLPathFound(t *testing.T) {
+	client := request.NewClient(request.NewClientOptions{
+		RateLimit: 500,
+	})
+	httpmock.ActivateNonDefault(client.Client)
+	defer httpmock.DeactivateAndReset()
+
+	operation := operation.MustNewOperation(http.MethodGet, "http://localhost:8080/.aws/credentials", nil, client)
+	httpmock.RegisterResponder(operation.Method, operation.URL.String(), httpmock.NewBytesResponder(http.StatusOK, nil))
+	httpmock.RegisterNoResponder(httpmock.NewBytesResponder(http.StatusNotFound, nil))
+
+	report, err := exposedfiles.ScanHandler(operation, auth.MustNewNoAuthSecurityScheme())
+
+	require.NoError(t, err)
+	assert.Greater(t, httpmock.GetTotalCallCount(), 0)
+	assert.True(t, report.Issues[0].HasFailed())
+}

scan/discover/utils.go

@@ -1,6 +1,7 @@
 package discover
 
 import (
+	"log"
 	"net/http"
 	"net/url"
 
@@ -11,7 +12,7 @@ import (
 	"github.com/cerberauth/vulnapi/seclist"
 )
 
-type DiscoverData struct {
+type DiscoverData []struct {
 	URL string
 }
 
@@ -56,32 +57,35 @@ func ScanURLs(scanUrls []string, op *operation.Operation, securityScheme *auth.S
 		}(chunk)
 	}
 
+	data := DiscoverData{}
 	for i := 0; i < len(scanUrls); i++ {
 		select {
 		case attempt := <-results:
 			r.AddScanAttempt(attempt)
 			if attempt.Response.GetStatusCode() == http.StatusOK { // TODO: check if the response contains the expected content
-				r.WithData(DiscoverData{
-					URL: attempt.Request.GetURL(),
-				}).AddIssueReport(vulnReport.Fail()).End()
-				return r, nil
+				data = append(data, struct{ URL string }{URL: attempt.Request.GetURL()})
 			}
 		case err := <-errors:
-			return r, err
+			log.Printf("Error scanning URL: %v", err)
+			continue
 		}
 	}
 
+	if len(data) > 0 {
+		r.WithData(data).AddIssueReport(vulnReport.Fail()).End()
+		return r, nil
+	}
+
 	r.AddIssueReport(vulnReport.Pass()).End()
 	return r, nil
 }
 
-func CreateURLScanHandler(name string, seclistUrl string, defaultUrls []string, r *report.ScanReport, vulnReport *report.IssueReport) func(operation *operation.Operation, securityScheme *auth.SecurityScheme) (*report.ScanReport, error) {
-	scanUrls := defaultUrls
-	if urlsFromSeclist, err := seclist.NewSecListFromURL(name, seclistUrl); err == nil && urlsFromSeclist != nil {
-		scanUrls = urlsFromSeclist.Items
+func DownloadAndScanURLs(name string, seclistUrl string, r *report.ScanReport, vulnReport *report.IssueReport, op *operation.Operation, securityScheme *auth.SecurityScheme) (*report.ScanReport, error) {
+	urlsFromSeclist, err := seclist.NewSecListFromURL(name, seclistUrl)
+	if err != nil {
+		return nil, err
 	}
+	scanUrls := urlsFromSeclist.Items
 
-	return func(op *operation.Operation, securityScheme *auth.SecurityScheme) (*report.ScanReport, error) {
-		return ScanURLs(scanUrls, op, securityScheme, r, vulnReport)
-	}
+	return ScanURLs(scanUrls, op, securityScheme, r, vulnReport)
 }