add report subcommand with custom templates (#141)

The new report subcommand replaces the existing csv command (and the csv
command is now deprecated). By default `go-licenses report` will output
CSV, but it also allows for a custom go template to be used.

Author: willnorris

README.md

@@ -39,7 +39,7 @@ If you were using `go get` to install this tool, note that
 ## Reports
 
 ```shell
-$ go-licenses csv github.com/google/go-licenses
+$ go-licenses report github.com/google/go-licenses
 W0410 06:02:57.077781   31529 library.go:86] "golang.org/x/sys/unix" contains non-Go code that can't be inspected for further dependencies:
 /home/username/go/pkg/mod/golang.org/x/[email protected]/unix/asm_linux_amd64.s
 W0410 06:02:59.476443   31529 library.go:86] "golang.org/x/crypto/curve25519/internal/field" contains non-Go code that can't be inspected for further dependencies:
@@ -83,21 +83,76 @@ share a license file.
 
 URLs are versioned based on go modules metadata.
 
-**Tip**: go-licenses writes CSV to stdout and info/warnings/errors logs to stderr.
-To save the CSV to a file `licenses.csv` in bash, run:
+**Tip**: go-licenses writes the report to stdout and info/warnings/errors logs
+to stderr. To save the CSV to a file `licenses.csv` in bash, run:
 
 ```bash
-go-licenses csv github.com/google/go-licenses > licenses.csv
+go-licenses report github.com/google/go-licenses > licenses.csv
 ```
 
 Or, to also save error logs to an `errors` file, run:
 
 ```bash
-go-licenses csv github.com/google/go-licenses > licenses.csv 2> errors
+go-licenses report github.com/google/go-licenses > licenses.csv 2> errors
 ```
 
 **Note**: some warnings and errors may be expected, refer to [Warnings and Errors](#warnings-and-errors) for more information.
 
+## Reports with Custom Templates
+
+```shell
+go-licenses report github.com/google/go-licenses --template testdata/modules/hello01/licenses.tpl
+W0822 16:56:50.696198   10200 library.go:94] "golang.org/x/sys/unix" contains non-Go code that can't be inspected for further dependencies:
+/Users/willnorris/go/pkg/mod/golang.org/x/[email protected]/unix/asm_bsd_arm64.s
+/Users/willnorris/go/pkg/mod/golang.org/x/[email protected]/unix/zsyscall_darwin_arm64.1_13.s
+/Users/willnorris/go/pkg/mod/golang.org/x/[email protected]/unix/zsyscall_darwin_arm64.s
+W0822 16:56:51.466449   10200 library.go:94] "golang.org/x/crypto/chacha20" contains non-Go code that can't be inspected for further dependencies:
+/Users/willnorris/go/pkg/mod/golang.org/x/[email protected]/chacha20/chacha_arm64.s
+W0822 16:56:51.475139   10200 library.go:94] "golang.org/x/crypto/curve25519/internal/field" contains non-Go code that can't be inspected for further dependencies:
+/Users/willnorris/go/pkg/mod/golang.org/x/[email protected]/curve25519/internal/field/fe_arm64.s
+W0822 16:56:51.602250   10200 library.go:269] module github.com/google/go-licenses has empty version, defaults to HEAD. The license URL may be incorrect. Please verify!
+W0822 16:56:51.605074   10200 library.go:269] module github.com/google/go-licenses has empty version, defaults to HEAD. The license URL may be incorrect. Please verify!
+
+ - github.com/emirpasic/gods ([BSD-2-Clause](https://github.com/emirpasic/gods/blob/v1.12.0/LICENSE))
+ - github.com/golang/glog ([Apache-2.0](https://github.com/golang/glog/blob/23def4e6c14b/LICENSE))
+ - github.com/golang/groupcache/lru ([Apache-2.0](https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE))
+ - github.com/google/go-licenses ([Apache-2.0](https://github.com/google/go-licenses/blob/HEAD/LICENSE))
+ - github.com/google/go-licenses/internal/third_party/pkgsite ([BSD-3-Clause](https://github.com/google/go-licenses/blob/HEAD/internal/third_party/pkgsite/LICENSE))
+ - github.com/google/licenseclassifier ([Apache-2.0](https://github.com/google/licenseclassifier/blob/3043a050f148/LICENSE))
+ - github.com/google/licenseclassifier/licenses ([Unlicense](https://github.com/google/licenseclassifier/blob/3043a050f148/licenses/Unlicense.txt))
+ - github.com/google/licenseclassifier/stringclassifier ([Apache-2.0](https://github.com/google/licenseclassifier/blob/3043a050f148/stringclassifier/LICENSE))
+ - github.com/jbenet/go-context/io ([MIT](https://github.com/jbenet/go-context/blob/d14ea06fba99/LICENSE))
+ - github.com/kevinburke/ssh_config ([MIT](https://github.com/kevinburke/ssh_config/blob/01f96b0aa0cd/LICENSE))
+ - github.com/mitchellh/go-homedir ([MIT](https://github.com/mitchellh/go-homedir/blob/v1.1.0/LICENSE))
+ - github.com/otiai10/copy ([MIT](https://github.com/otiai10/copy/blob/v1.6.0/LICENSE))
+ - github.com/sergi/go-diff/diffmatchpatch ([MIT](https://github.com/sergi/go-diff/blob/v1.2.0/LICENSE))
+ - github.com/spf13/cobra ([Apache-2.0](https://github.com/spf13/cobra/blob/v1.5.0/LICENSE.txt))
+ - github.com/spf13/pflag ([BSD-3-Clause](https://github.com/spf13/pflag/blob/v1.0.5/LICENSE))
+ - github.com/src-d/gcfg ([BSD-3-Clause](https://github.com/src-d/gcfg/blob/v1.4.0/LICENSE))
+ - github.com/xanzy/ssh-agent ([Apache-2.0](https://github.com/xanzy/ssh-agent/blob/v0.2.1/LICENSE))
+ - go.opencensus.io ([Apache-2.0](https://github.com/census-instrumentation/opencensus-go/blob/v0.23.0/LICENSE))
+ - golang.org/x/crypto ([BSD-3-Clause](https://cs.opensource.google/go/x/crypto/+/5e0467b6:LICENSE))
+ - golang.org/x/mod/semver ([BSD-3-Clause](https://cs.opensource.google/go/x/mod/+/86c51ed2:LICENSE))
+ - golang.org/x/net ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/a158d28d:LICENSE))
+ - golang.org/x/sys ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/8c9f86f7:LICENSE))
+ - golang.org/x/tools ([BSD-3-Clause](https://cs.opensource.google/go/x/tools/+/v0.1.12:LICENSE))
+ - gopkg.in/src-d/go-billy.v4 ([Apache-2.0](https://github.com/src-d/go-billy/blob/v4.3.2/LICENSE))
+ - gopkg.in/src-d/go-git.v4 ([Apache-2.0](https://github.com/src-d/go-git/blob/v4.13.1/LICENSE))
+ - gopkg.in/warnings.v0 ([BSD-2-Clause](https://github.com/go-warnings/warnings/blob/v0.1.2/LICENSE))
+```
+
+This command executes a specified Go template file to generate a report of
+licenses.  The template file is passed a slice of structs containing license
+data:
+
+```go
+[]struct {
+  Name        string
+  LicenseURL  string
+  LicenseName string
+}
+```
+
 ## Save licenses, copyright notices and source code (depending on license type)
 
 ```shell
@@ -124,10 +179,16 @@ for licenses considered forbidden.
 
 ## Usages
 
-Report usage:
+Report usage (default csv output):
+
+```shell
+go-licenses report <package> [package...]
+```
+
+Report usage (using custom template file):
 
 ```shell
-go-licenses csv <package> [package...]
+go-licenses report <package> [package...] --template=<template_file>
 ```
 
 Save licenses, copyright notices and source code (depending on license type):
@@ -159,7 +220,7 @@ To read dependencies from packages with
 `$GOFLAGS` environment variable.
 
 ```shell
-$ GOFLAGS="-tags=tools" go-licenses csv google.golang.org/grpc/test/tools
+$ GOFLAGS="-tags=tools" go-licenses report google.golang.org/grpc/test/tools
 github.com/BurntSushi/toml,https://github.com/BurntSushi/toml/blob/master/COPYING,MIT
 google.golang.org/grpc/test/tools,Unknown,Apache-2.0
 honnef.co/go/tools/lint,Unknown,BSD-3-Clause
@@ -183,7 +244,7 @@ $ go-licenses check \
 ```
 
 Note that dependencies from the ignored packages are still resolved and checked.
-This flag makes effect to `check`, `csv` and `save` commands.
+This flag makes effect to `check`, `report` and `save` commands.
 
 ## Warnings and errors
 

csv.go

@@ -15,67 +15,25 @@
 package main
 
 import (
-	"context"
-	"encoding/csv"
-	"os"
-
-	"github.com/golang/glog"
-	"github.com/google/go-licenses/licenses"
 	"github.com/spf13/cobra"
 )
 
 var (
-	csvHelp = "Prints all licenses that apply to one or more Go packages and their dependencies."
+	csvHelp = "Prints all licenses that apply to one or more Go packages and their dependencies. (Deprecated: use report instead)"
 	csvCmd  = &cobra.Command{
 		Use:   "csv <package> [package...]",
 		Short: csvHelp,
 		Long:  csvHelp + packageHelp,
 		Args:  cobra.MinimumNArgs(1),
 		RunE:  csvMain,
 	}
-
-	gitRemotes []string
 )
 
 func init() {
-	csvCmd.Flags().StringArrayVar(&gitRemotes, "git_remote", []string{"origin", "upstream"}, "Remote Git repositories to try")
-
 	rootCmd.AddCommand(csvCmd)
 }
 
 func csvMain(_ *cobra.Command, args []string) error {
-	writer := csv.NewWriter(os.Stdout)
-
-	classifier, err := licenses.NewClassifier(confidenceThreshold)
-	if err != nil {
-		return err
-	}
-
-	libs, err := licenses.Libraries(context.Background(), classifier, ignore, args...)
-	if err != nil {
-		return err
-	}
-	for _, lib := range libs {
-		licenseURL := "Unknown"
-		licenseName := "Unknown"
-		if lib.LicensePath != "" {
-			name, _, err := classifier.Identify(lib.LicensePath)
-			if err == nil {
-				licenseName = name
-			} else {
-				glog.Errorf("Error identifying license in %q: %v", lib.LicensePath, err)
-			}
-			url, err := lib.FileURL(context.Background(), lib.LicensePath)
-			if err == nil {
-				licenseURL = url
-			} else {
-				glog.Warningf("Error discovering license URL: %s", err)
-			}
-		}
-		if err := writer.Write([]string{lib.Name(), licenseURL, licenseName}); err != nil {
-			return err
-		}
-	}
-	writer.Flush()
-	return writer.Error()
+	// without a --template flag, reportMain will output CSV
+	return reportMain(nil, args)
 }

e2e_test.go

@@ -29,17 +29,26 @@ import (
 
 var update = flag.Bool("update", false, "update golden files")
 
-func TestCsvCommandE2E(t *testing.T) {
-	workdirs := []string{
-		"testdata/modules/hello01",
-		"testdata/modules/cli02",
-		"testdata/modules/vendored03",
-		"testdata/modules/replace04",
+func TestReportCommandE2E(t *testing.T) {
+	tests := []struct {
+		workdir        string
+		args           []string // additional arguments to pass to report command.
+		goldenFilePath string
+	}{
+		{"testdata/modules/hello01", nil, "licenses.csv"},
+		{"testdata/modules/cli02", nil, "licenses.csv"},
+		{"testdata/modules/vendored03", nil, "licenses.csv"},
+		{"testdata/modules/replace04", nil, "licenses.csv"},
+
+		{"testdata/modules/hello01", []string{"--template", "licenses.tpl"}, "licenses.md"},
 	}
+
 	originalWorkDir, err := os.Getwd()
 	if err != nil {
 		t.Fatal(err)
 	}
+	t.Cleanup(func() { _ = os.Chdir(originalWorkDir) })
+
 	// This builds go-licenses CLI to temporary dir.
 	tempDir, err := ioutil.TempDir("", "")
 	if err != nil {
@@ -53,9 +62,10 @@ func TestCsvCommandE2E(t *testing.T) {
 		t.Fatal(err)
 	}
 	t.Logf("Built go-licenses binary in %s.", goLicensesPath)
-	for _, workdir := range workdirs {
-		t.Run(workdir, func(t *testing.T) {
-			err := os.Chdir(filepath.Join(originalWorkDir, workdir))
+
+	for _, tt := range tests {
+		t.Run(tt.workdir, func(t *testing.T) {
+			err := os.Chdir(filepath.Join(originalWorkDir, tt.workdir))
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -64,25 +74,25 @@ func TestCsvCommandE2E(t *testing.T) {
 			if err != nil {
 				t.Fatalf("downloading go modules:\n%s", string(log))
 			}
-			cmd = exec.Command(goLicensesPath, "csv", ".")
+			args := append([]string{"report", "."}, tt.args...)
+			cmd = exec.Command(goLicensesPath, args...)
 			// Capture stderr to buffer.
 			var stderr bytes.Buffer
 			cmd.Stderr = &stderr
-			t.Logf("%s $ go-licenses csv .", workdir)
+			t.Logf("%s $ go-licenses csv .", tt.workdir)
 			output, err := cmd.Output()
 			if err != nil {
 				t.Logf("\n=== start of log ===\n%s=== end of log ===\n\n\n", stderr.String())
 				t.Fatalf("running go-licenses csv: %s. Full log shown above.", err)
 			}
 			got := string(output)
-			goldenFilePath := "licenses.csv"
 			if *update {
-				err := ioutil.WriteFile(goldenFilePath, output, 0600)
+				err := ioutil.WriteFile(tt.goldenFilePath, output, 0600)
 				if err != nil {
 					t.Fatalf("writing golden file: %s", err)
 				}
 			}
-			goldenBytes, err := ioutil.ReadFile(goldenFilePath)
+			goldenBytes, err := ioutil.ReadFile(tt.goldenFilePath)
 			if err != nil {
 				if errors.Is(err, os.ErrNotExist) {
 					t.Fatalf("reading golden file: %s. Create a golden file by running `go test --update .`", err)

report.go

@@ -0,0 +1,117 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"encoding/csv"
+	"io/ioutil"
+	"os"
+	"text/template"
+
+	"github.com/golang/glog"
+	"github.com/google/go-licenses/licenses"
+	"github.com/spf13/cobra"
+)
+
+var (
+	reportHelp = "Prints report of all licenses that apply to one or more Go packages and their dependencies."
+	reportCmd  = &cobra.Command{
+		Use:   "report <package> [package...]",
+		Short: reportHelp,
+		Long:  reportHelp + packageHelp,
+		Args:  cobra.MinimumNArgs(1),
+		RunE:  reportMain,
+	}
+
+	templateFile string
+)
+
+func init() {
+	reportCmd.Flags().StringVar(&templateFile, "template", "", "Custom Go template file to use for report")
+
+	rootCmd.AddCommand(reportCmd)
+}
+
+type libraryData struct {
+	Name        string
+	LicenseURL  string
+	LicenseName string
+}
+
+func reportMain(_ *cobra.Command, args []string) error {
+	classifier, err := licenses.NewClassifier(confidenceThreshold)
+	if err != nil {
+		return err
+	}
+
+	libs, err := licenses.Libraries(context.Background(), classifier, ignore, args...)
+	if err != nil {
+		return err
+	}
+
+	var reportData []libraryData
+	for _, lib := range libs {
+		libData := libraryData{
+			Name:        lib.Name(),
+			LicenseURL:  "Unknown",
+			LicenseName: "Unknown",
+		}
+		if lib.LicensePath != "" {
+			name, _, err := classifier.Identify(lib.LicensePath)
+			if err == nil {
+				libData.LicenseName = name
+			} else {
+				glog.Errorf("Error identifying license in %q: %v", lib.LicensePath, err)
+			}
+			url, err := lib.FileURL(context.Background(), lib.LicensePath)
+			if err == nil {
+				libData.LicenseURL = url
+			} else {
+				glog.Warningf("Error discovering license URL: %s", err)
+			}
+		}
+		reportData = append(reportData, libData)
+	}
+
+	if templateFile == "" {
+		return reportCSV(reportData)
+	} else {
+		return reportTemplate(reportData)
+	}
+}
+
+func reportCSV(libs []libraryData) error {
+	writer := csv.NewWriter(os.Stdout)
+	for _, lib := range libs {
+		if err := writer.Write([]string{lib.Name, lib.LicenseURL, lib.LicenseName}); err != nil {
+			return err
+		}
+	}
+	writer.Flush()
+	return writer.Error()
+}
+
+func reportTemplate(libs []libraryData) error {
+	templateBytes, err := ioutil.ReadFile(templateFile)
+	if err != nil {
+		return err
+	}
+	tmpl, err := template.New("").Parse(string(templateBytes))
+	if err != nil {
+		return err
+	}
+	return tmpl.Execute(os.Stdout, libs)
+}

testdata/modules/hello01/licenses.md

@@ -0,0 +1,2 @@
+
+ - github.com/google/go-licenses/testdata/modules/hello01 ([Apache-2.0](https://github.com/google/go-licenses/blob/HEAD/testdata/modules/hello01/LICENSE))