tests(pypi): add tests for fetching package from pypi

internal/helper/gateway.go

@@ -9,16 +9,18 @@ import (
 )
 
 type Client struct {
-	HTTP *http.Client
+	HTTP    *http.Client
+	BaseURL string
 }
 
 // NewClient
 // todo: complete proper client settings
-func NewClient() *Client {
+func NewClient(baseURL string) *Client {
 	return &Client{
 		HTTP: &http.Client{
 			Timeout: time.Second * 5,
 		},
+		BaseURL: baseURL,
 	}
 }
 
@@ -42,3 +44,13 @@ func (c *Client) CheckURL(url string) bool {
 
 	return r.StatusCode == http.StatusOK
 }
+
+// Get makes a GET request to the specified url and returns the response.
+func (c *Client) Get(url string) (*http.Response, error) {
+	r, err := c.HTTP.Get(url)
+	if err != nil {
+		return nil, err
+	}
+
+	return r, nil
+}

pip/pipenv/handler.go

@@ -20,6 +20,7 @@ const (
 	placeholderPkgName  = "{PACKAGE}"
 	packageSrcLocation  = "/src/"
 	packageSiteLocation = "/site-packages"
+	pypiURL             = "https://pypi.org"
 )
 
 var (
@@ -113,7 +114,8 @@ func (m *PipEnv) ListUsedModules(path string) ([]meta.Package, error) {
 		return m.allModules, errFailedToConvertModules
 	}
 
-	decoder := worker.NewMetadataDecoder(m.GetPackageDetails)
+	pypiDataFactory := worker.NewPypiPackageDataFactory(helper.NewClient(pypiURL))
+	decoder := worker.NewMetadataDecoder(m.GetPackageDetails, pypiDataFactory)
 	metainfo, err := decoder.ConvertMetadataToModules(m.pkgs, &m.allModules)
 	if err != nil {
 		return m.allModules, err

pip/worker/decoder.go

@@ -19,30 +19,32 @@ type GetPackageDetailsFunc = func(PackageName string) (string, error)
 
 type MetadataDecoder struct {
 	getPkgDetailsFunc GetPackageDetailsFunc
+	pf                PypiPackageDataFactory
 }
 
-// New Metadata Decoder ...
-func NewMetadataDecoder(pkgDetailsFunc GetPackageDetailsFunc) *MetadataDecoder {
+// NewMetadataDecoder ...
+func NewMetadataDecoder(pkgDetailsFunc GetPackageDetailsFunc, pf PypiPackageDataFactory) *MetadataDecoder {
 	return &MetadataDecoder{
 		getPkgDetailsFunc: pkgDetailsFunc,
+		pf:                pf,
 	}
 }
 
-func SetMetadataValues(matadata *Metadata, datamap map[string]string) {
-	matadata.Name = datamap[KeyName]
-	matadata.Version = datamap[KeyVersion]
-	matadata.Description = datamap[KeySummary]
-	matadata.HomePage = datamap[KeyHomePage]
-	matadata.Author = datamap[KeyAuthor]
-	matadata.AuthorEmail = datamap[KeyAuthorEmail]
-	matadata.License = datamap[KeyLicense]
-	matadata.Location = datamap[KeyLocation]
+func SetMetadataValues(metadata *Metadata, datamap map[string]string) {
+	metadata.Name = datamap[KeyName]
+	metadata.Version = datamap[KeyVersion]
+	metadata.Description = datamap[KeySummary]
+	metadata.HomePage = datamap[KeyHomePage]
+	metadata.Author = datamap[KeyAuthor]
+	metadata.AuthorEmail = datamap[KeyAuthorEmail]
+	metadata.License = datamap[KeyLicense]
+	metadata.Location = datamap[KeyLocation]
 
 	// Parsing "Requires"
 	if len(datamap[KeyRequires]) != 0 {
-		matadata.Modules = strings.Split(datamap[KeyRequires], ",")
-		for i, v := range matadata.Modules {
-			matadata.Modules[i] = strings.TrimSpace(v)
+		metadata.Modules = strings.Split(datamap[KeyRequires], ",")
+		for i, v := range metadata.Modules {
+			metadata.Modules[i] = strings.TrimSpace(v)
 		}
 	}
 }
@@ -144,7 +146,7 @@ func (d *MetadataDecoder) BuildModule(metadata Metadata) meta.Package {
 		module.PackageURL = metadata.HomePage
 	}
 
-	pypiData, err := GetPackageDataFromPyPi(metadata.PackageJSONURL)
+	pypiData, err := d.pf.GetPackageData(metadata.PackageJSONURL)
 	if err != nil {
 		log.Warnf("Unable to get `%s` package details from pypi.org", metadata.Name)
 		if (len(metadata.HomePage) > 0) && (metadata.HomePage != "None") {
@@ -154,7 +156,7 @@ func (d *MetadataDecoder) BuildModule(metadata Metadata) meta.Package {
 
 	// Prepare supplier contact
 	if len(metadata.Author) > 0 && metadata.Author == "None" {
-		metadata.Author, metadata.AuthorEmail = GetMaintenerDataFromPyPiPackageData(pypiData)
+		metadata.Author, metadata.AuthorEmail = d.pf.GetMaintainerData(pypiData)
 	}
 
 	contactType := meta.Person
@@ -169,11 +171,11 @@ func (d *MetadataDecoder) BuildModule(metadata Metadata) meta.Package {
 	}
 
 	// Prepare checksum
-	checksum := GetChecksumeFromPyPiPackageData(pypiData, metadata)
+	checksum := d.pf.GetChecksum(pypiData, metadata)
 	module.Checksum = *checksum
 
 	// Prepare download location
-	downloadURL := GetDownloadLocationFromPyPiPackageData(pypiData, metadata)
+	downloadURL := d.pf.GetDownloadLocationFromPyPiPackageData(pypiData, metadata)
 	module.PackageDownloadLocation = downloadURL
 	if len(downloadURL) == 0 {
 		if metadata.Root {

pip/worker/pypi.go

@@ -5,11 +5,13 @@ package worker
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"reflect"
 	"strings"
 
+	"github.com/opensbom-generator/parsers/internal/helper"
 	"github.com/opensbom-generator/parsers/meta"
 )
 
@@ -82,32 +84,36 @@ var HashAlgoPickOrder []meta.HashAlgorithm = []meta.HashAlgorithm{
 	meta.HashAlgoMD2,
 }
 
-func makeGetRequest(packageJSONURL string) (*http.Response, error) {
-	url := "https://" + packageJSONURL
-
-	request, _ := http.NewRequest("GET", url, nil)
-	request.Header.Set("Accept", "application/json")
+type pypiPackageDataFactory struct {
+	client *helper.Client
+}
 
-	client := &http.Client{}
-	response, err := client.Do(request)
-	if err != nil {
-		return nil, err
-	}
+type PypiPackageDataFactory interface {
+	GetPackageData(packageJSONURL string) (PypiPackageData, error)
+	GetMaintainerData(pkgData PypiPackageData) (string, string)
+	GetChecksum(pkgData PypiPackageData, metadata Metadata) *meta.Checksum
+	GetDownloadLocationFromPyPiPackageData(pkgData PypiPackageData, metadata Metadata) string
+}
 
-	if response.StatusCode != http.StatusOK {
-		return nil, errorPypiCouldNotFetchPkgData
+// NewPypiPackageDataFactory ...
+func NewPypiPackageDataFactory(client *helper.Client) PypiPackageDataFactory {
+	return &pypiPackageDataFactory{
+		client: client,
 	}
-
-	return response, err
 }
 
-func GetPackageDataFromPyPi(packageJSONURL string) (PypiPackageData, error) {
+func (pf *pypiPackageDataFactory) GetPackageData(packageJSONURL string) (PypiPackageData, error) {
 	packageInfo := PypiPackageData{}
 
-	response, err := makeGetRequest(packageJSONURL)
+	packageJSONURL = strings.Replace(packageJSONURL, "pypi.org", "", 1)
+	response, err := pf.client.HTTP.Get(fmt.Sprintf("%s%s", pf.client.BaseURL, packageJSONURL))
 	if err != nil {
 		return packageInfo, err
 	}
+
+	if response.StatusCode != http.StatusOK {
+		return packageInfo, errorPypiCouldNotFetchPkgData
+	}
 	defer response.Body.Close()
 
 	jsondata, _ := io.ReadAll(response.Body)
@@ -119,7 +125,7 @@ func GetPackageDataFromPyPi(packageJSONURL string) (PypiPackageData, error) {
 	return packageInfo, nil
 }
 
-func GetMaintenerDataFromPyPiPackageData(pkgData PypiPackageData) (string, string) {
+func (pf *pypiPackageDataFactory) GetMaintainerData(pkgData PypiPackageData) (string, string) {
 	var name string
 	var email string
 	if len(pkgData.Info.Maintainer) > 0 {
@@ -131,27 +137,54 @@ func GetMaintenerDataFromPyPiPackageData(pkgData PypiPackageData) (string, strin
 	return name, email
 }
 
-func GetHighestOrderHashData(digests DigestTypes) (meta.HashAlgorithm, string) {
-	var algoType meta.HashAlgorithm
-	var digestValue string
+func (pf *pypiPackageDataFactory) GetChecksum(pkgData PypiPackageData, metadata Metadata) *meta.Checksum {
+	checksum := meta.Checksum{
+		Algorithm: meta.HashAlgoSHA1,
+		Content:   []byte(pkgData.Info.Name),
+	}
 
-	v := reflect.ValueOf(digests)
-	for _, algo := range HashAlgoPickOrder {
-		f := v.FieldByName(string(algo))
-		if f.IsValid() {
-			algoType = algo
-			digestValue = f.String()
-			return algoType, digestValue
+	for _, packageDistInfo := range pkgData.Urls {
+		distInfo, status := getPackageBDistWheelInfo(packageDistInfo, metadata.Generator, metadata.Tag, metadata.CPVersion)
+		if status {
+			algo, value := getHighestOrderHashData(distInfo.Digests)
+			checksum.Algorithm = algo
+			checksum.Value = value
+			return &checksum
+		}
+
+		distInfo, status = getPackageSDistInfo(packageDistInfo, "sdist")
+		if status {
+			algo, value := getHighestOrderHashData(distInfo.Digests)
+			checksum.Algorithm = algo
+			checksum.Value = value
+			return &checksum
 		}
 	}
 
-	return algoType, digestValue
+	return &checksum
+}
+
+func (pf *pypiPackageDataFactory) GetDownloadLocationFromPyPiPackageData(pkgData PypiPackageData, metadata Metadata) string {
+	for _, packageDistInfo := range pkgData.Urls {
+		distInfo, status := getPackageBDistWheelInfo(packageDistInfo, metadata.Generator, metadata.Tag, metadata.CPVersion)
+		if status {
+			return distInfo.URL
+		}
+
+		distInfo, status = getPackageSDistInfo(packageDistInfo, "sdist")
+		if status {
+			return distInfo.URL
+		}
+	}
+
+	return ""
 }
 
-func GetPackageBDistWheelInfo(distInfo PypiPackageDistInfo, generator string, tag string, cpversion string) (PypiPackageDistInfo, bool) {
+func getPackageBDistWheelInfo(distInfo PypiPackageDistInfo, generator string,
+	tag string, cpVersion string) (PypiPackageDistInfo, bool) {
 	PackageType := strings.EqualFold(distInfo.PackageType, generator)
 	Tag := strings.Contains(strings.ToLower(distInfo.Filename), strings.ToLower(tag))
-	CPVersion := strings.EqualFold(distInfo.PythonVersion, cpversion)
+	CPVersion := strings.EqualFold(distInfo.PythonVersion, cpVersion)
 	Py2Py3 := strings.Contains(strings.ToLower("py2.py3"), strings.ToLower(distInfo.PythonVersion))
 
 	status := false
@@ -163,7 +196,7 @@ func GetPackageBDistWheelInfo(distInfo PypiPackageDistInfo, generator string, ta
 	return distInfo, status
 }
 
-func GetPackageSDistInfo(distInfo PypiPackageDistInfo, generator string) (PypiPackageDistInfo, bool) {
+func getPackageSDistInfo(distInfo PypiPackageDistInfo, generator string) (PypiPackageDistInfo, bool) {
 	PackageType := strings.EqualFold(distInfo.PackageType, generator)
 	Source := strings.EqualFold(distInfo.PythonVersion, "source")
 
@@ -176,45 +209,19 @@ func GetPackageSDistInfo(distInfo PypiPackageDistInfo, generator string) (PypiPa
 	return distInfo, status
 }
 
-func GetChecksumeFromPyPiPackageData(pkgData PypiPackageData, metadata Metadata) *meta.Checksum {
-	checksum := meta.Checksum{
-		Algorithm: meta.HashAlgoSHA1,
-		Content:   []byte(pkgData.Info.Name),
-	}
-
-	for _, packageDistInfo := range pkgData.Urls {
-		distInfo, status := GetPackageBDistWheelInfo(packageDistInfo, metadata.Generator, metadata.Tag, metadata.CPVersion)
-		if status {
-			algo, value := GetHighestOrderHashData(distInfo.Digests)
-			checksum.Algorithm = algo
-			checksum.Value = value
-			return &checksum
-		}
-
-		distInfo, status = GetPackageSDistInfo(packageDistInfo, "sdist")
-		if status {
-			algo, value := GetHighestOrderHashData(distInfo.Digests)
-			checksum.Algorithm = algo
-			checksum.Value = value
-			return &checksum
-		}
-	}
-
-	return &checksum
-}
-
-func GetDownloadLocationFromPyPiPackageData(pkgData PypiPackageData, metadata Metadata) string {
-	for _, packageDistInfo := range pkgData.Urls {
-		distInfo, status := GetPackageBDistWheelInfo(packageDistInfo, metadata.Generator, metadata.Tag, metadata.CPVersion)
-		if status {
-			return distInfo.URL
-		}
+func getHighestOrderHashData(digests DigestTypes) (meta.HashAlgorithm, string) {
+	var algoType meta.HashAlgorithm
+	var digestValue string
 
-		distInfo, status = GetPackageSDistInfo(packageDistInfo, "sdist")
-		if status {
-			return distInfo.URL
+	v := reflect.ValueOf(digests)
+	for _, algo := range HashAlgoPickOrder {
+		f := v.FieldByName(string(algo))
+		if f.IsValid() {
+			algoType = algo
+			digestValue = f.String()
+			return algoType, digestValue
 		}
 	}
 
-	return ""
+	return algoType, digestValue
 }

pip/worker/pypi_test.go

@@ -0,0 +1,48 @@
+package worker
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/opensbom-generator/parsers/internal/helper"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetPackageDataFromPyPi(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch strings.TrimSpace(r.URL.Path) {
+		case "/pypi/requests/jso":
+			byteData, err := os.ReadFile("../testdata/requests_pypi_data.json")
+			if err != nil {
+				panic(err)
+			}
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusOK)
+			w.Write(byteData)
+		default:
+			http.NotFoundHandler().ServeHTTP(w, r)
+		}
+	}))
+	defer server.Close()
+
+	for name, tc := range map[string]struct {
+		packageJSONUrl string
+		expectedErr    error
+	}{
+		"valid package url": {
+			packageJSONUrl: "/pypi/requests/jso",
+			expectedErr:    nil,
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			mockClient := helper.NewClient(server.URL)
+			factory := NewPypiPackageDataFactory(mockClient)
+			packageInfo, err := factory.GetPackageData(tc.packageJSONUrl)
+			require.ErrorIs(t, tc.expectedErr, err)
+			require.NotNil(t, packageInfo)
+		})
+	}
+}