Merge pull request #143 from bpicode/Issue-124-deb-copyright

Write debian copyright file by parsing NOTICE and vendor tree

.gitignore

@@ -5,6 +5,7 @@
 *.out
 coverage-all.html
 /fritzctl
+/notice2copyright
 
 # Folders
 _obj

Makefile

@@ -6,13 +6,14 @@ FRITZCTL_OUTPUT           ?= fritzctl
 FRITZCTL_REVISION         := $(shell git rev-parse HEAD)
 BASH_COMPLETION_OUTPUT    ?= "os/completion/fritzctl"
 MAN_PAGE_OUTPUT           ?= "os/man/fritzctl.1"
+COPYRIGHT_OUTPUT          ?= "os/doc/copyright"
 DEPENDENCIES_GRAPH_OUTPUT ?= "dependencies.png"
 BUILDFLAGS                := -ldflags="-s -w -X github.com/bpicode/fritzctl/config.Version=$(FRITZCTL_VERSION) -X github.com/bpicode/fritzctl/config.Revision=$(FRITZCTL_REVISION)" -gcflags="-trimpath=$(GOPATH)" -asmflags="-trimpath=$(GOPATH)"
 TESTFLAGS                 ?=
 
-all: sysinfo build install test codequality completion_bash man
+all: sysinfo build install test codequality completion_bash man copyright
 
-.PHONY: clean build man
+.PHONY: clean build man copyright
 
 define ok
 	@tput setaf 6 2>/dev/null || echo -n ""
@@ -36,6 +37,7 @@ clean:
 	@go clean -i
 	@rm -f ./os/completion/fritzctl
 	@rm -f ./os/man/*.gz
+	@rm -f ./os/doc/copyright
 	@rm -f ./coverage-all.html
 	@rm -f ./coverage-all.out
 	@rm -f ./coverage.out
@@ -96,6 +98,12 @@ man:
 	@gzip --force $(MAN_PAGE_OUTPUT)
 	@$(call ok)
 
+copyright:
+	@echo -n ">> COPYRIGHT, output = $(COPYRIGHT_OUTPUT)"
+	@go build github.com/bpicode/fritzctl/tools/notice2copyright
+	@./notice2copyright ./ "MIT License (Expat)"> $(COPYRIGHT_OUTPUT)
+	@$(call ok)
+
 codequality:
 	@echo ">> CODE QUALITY"
 	@echo -n "     FMT"
@@ -179,15 +187,17 @@ pkg_darwin: dist_darwin
 	@zip -q build/distributions/fritzctl-$(FRITZCTL_VERSION)-darwin-amd64.zip build/distributions/darwin_amd64/fritzctl
 	@$(call ok)
 
-pkg_linux: dist_linux man completion_bash
+pkg_linux: dist_linux man completion_bash copyright
 	@mkdir -p build/distributions/linux_amd64/usr/bin
 	@mkdir -p build/distributions/linux_amd64/etc/fritzctl
 	@mkdir -p build/distributions/linux_amd64/etc/bash_completion.d
 	@mkdir -p build/distributions/linux_amd64/usr/share/man/man1
+	@mkdir -p build/distributions/linux_amd64/usr/share/doc/fritzctl
 	@cp os/completion/fritzctl build/distributions/linux_amd64/etc/bash_completion.d/
 	@cp os/config/fritzctl.json build/distributions/linux_amd64/etc/fritzctl/
 	@cp os/config/fritz.pem build/distributions/linux_amd64/etc/fritzctl/
 	@cp os/man/*.1.gz build/distributions/linux_amd64/usr/share/man/man1/
+	@cp os/doc/copyright build/distributions/linux_amd64/usr/share/doc/fritzctl/
 
 	@echo ">> PACKAGE, linux/amd64/deb"
 	@echo -n "     "
@@ -200,10 +210,12 @@ pkg_linux: dist_linux man completion_bash
 	@mkdir -p build/distributions/linux_arm/etc/fritzctl
 	@mkdir -p build/distributions/linux_arm/etc/bash_completion.d
 	@mkdir -p build/distributions/linux_arm/usr/share/man/man1
+	@mkdir -p build/distributions/linux_arm/usr/share/doc/fritzctl
 	@cp os/completion/fritzctl build/distributions/linux_arm/etc/bash_completion.d/
 	@cp os/config/fritzctl.json build/distributions/linux_arm/etc/fritzctl/
 	@cp os/config/fritz.pem build/distributions/linux_arm/etc/fritzctl/
 	@cp os/man/*.1.gz build/distributions/linux_arm/usr/share/man/man1/
+	@cp os/doc/copyright build/distributions/linux_arm/usr/share/doc/fritzctl/
 
 	@echo ">> PACKAGE, linux/armhf/deb"
 	@echo -n "     "

NOTICE

@@ -10,7 +10,7 @@ The following software have components provided under the terms of this license:
 
 - cobra (from https://github.com/spf13/cobra)
 - mousetrap (from https://github.com/inconshreveable/mousetrap)
-- yaml (from https://github.com/go-yaml/yaml)
+- yaml (from https://gopkg.in/yaml.v2)
 
 ========================================================================
 MIT License (Expat)
@@ -38,11 +38,11 @@ BSD 3-Clause License (Revised)
 
 The following software have components provided under the terms of this license:
 
-- crypto (from https://github.com/golang/crypto)
+- crypto (from https://golang.org/x/crypto)
 - httprouter (from https://github.com/julienschmidt/httprouter)
 - go-difflib (from https://github.com/pmezard/go-difflib)
 - pflag (from https://github.com/spf13/pflag)
-- sys (from https://github.com/golang/sys)
+- sys (from https://golang.org/x/sys)
 
 ========================================================================
 BSD 2-Clause License (FreeBSD/Simplified)

os/doc/.gitignore

@@ -0,0 +1 @@
+copyright

tools/README.md

@@ -0,0 +1 @@
+# Tools of our trade

tools/notice2copyright/README.md

@@ -0,0 +1,47 @@
+# notice2copyright
+
+## About
+
+Parse a `NOTICE` file to generate the [debian copyright file](https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/).
+
+## Build
+
+```sh
+go build
+```
+
+## Usage
+
+```sh
+notice2copyright path/to/github.com/user/project "MIT License (Expat)"
+```
+
+## Assumptions
+
+* Every dependency is licensed under one license known to this tool.
+* The project has a `NOTICE` file in the following format:
+  ```text
+  ...
+  
+  ========================================================================
+  Apache License 2.0 (Apache-2.0)
+  ========================================================================
+  
+  The following software have components provided under the terms of this license:
+  
+  - cobra (from https://github.com/spf13/cobra)
+
+  ========================================================================
+  MIT License (Expat)
+  ========================================================================
+  
+  The following software have components provided under the terms of this license:
+  
+  - color (from https://github.com/fatih/color)
+
+  ...
+  ```
+  The file needs to reside at the top-level directory of the project.
+* Vendoring. All dependencies need to be in the vendor folder including their `LICENSE` files.
+  A dependency referenced via `- cobra (from https://github.com/spf13/cobra)` in the `NOTICE`
+  file has to be present in the vendor folder as `vendor/github.com/spf13/cobra)`.

tools/notice2copyright/assert.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"fmt"
+	"log"
+)
+
+func assertOrFatal(val bool, f string, v ...interface{}) {
+	assertTrue(log.Fatalln, val, fmt.Sprintf(f, v...))
+}
+
+func assertTrue(fat func(v ...interface{}), val bool, f string) {
+	if !val {
+		fat(f)
+	}
+}

tools/notice2copyright/assert_test.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"log"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_assertTrue(t *testing.T) {
+	assert.NotPanics(t, func() {
+		assertTrue(log.Fatalln, true, "err")
+	})
+	assert.Panics(t, func() {
+		assertTrue(func(v ...interface{}) {
+			panic("panic")
+		}, false, "err")
+	})
+}

tools/notice2copyright/copyright_holder.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+var copyrightRegex = regexp.MustCompile(`^>*\s*[Cc]opyright\s(\([Cc]\)|©)*(?P<CLINE>.*)`)
+
+var copyrightNoAuthorRegex = regexp.MustCompile(`([Cc]opyright notice|[Cc]opyright license to reproduce)`)
+
+func findCopyrightHolders(projects []project, dir string) []project {
+	withCpHolders := make([]project, len(projects))
+	for i, p := range projects {
+		withCpHolders[i] = findCopyrightHoldersOf(p, dir)
+	}
+	return withCpHolders
+}
+
+func findCopyrightHoldersOf(p project, dir string) project {
+	srcDir := filepath.Join(dir, p.dir())
+	lf, err := openFirst(srcDir, "LICENSE", "LICENSE.md", "LICENSE.txt", "LICENSE.rst")
+	assertOrFatal(err == nil, "unable to open license file for project '%s' (%s): %v", p.name, p.url, err)
+	defer lf.Close()
+	p.copyrightHolders = readCopyrightHolders(lf)
+	return p
+}
+
+func openFirst(dir string, names ...string) (*os.File, error) {
+	for _, name := range names {
+		f, err := os.Open(filepath.Join(dir, name))
+		if err == nil {
+			return f, nil
+		}
+	}
+	return nil, fmt.Errorf("no license file [%s] found in directory '%s'", names, dir)
+}
+
+func readCopyrightHolders(r io.Reader) []string {
+	var holders []string
+	s := bufio.NewScanner(r)
+	for s.Scan() {
+		line := s.Text()
+		holders = addCopyRightHolder(line, holders)
+	}
+	if len(holders) == 0 {
+		holders = append(holders, "unknown")
+	}
+	return holders
+}
+
+func addCopyRightHolder(text string, holders []string) []string {
+	if copyrightRegex.MatchString(text) && !copyrightNoAuthorRegex.MatchString(text) {
+		holders = append(holders, parseCopyrightHolder(text))
+	}
+	return holders
+}
+
+func parseCopyrightHolder(text string) string {
+	matches := copyrightRegex.FindStringSubmatch(text)
+	return strings.TrimSpace(matches[len(matches)-1])
+}

tools/notice2copyright/copyright_holder_test.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_openFirst(t *testing.T) {
+	_, err := openFirst("/that/should/not/work/ever", "asdfagag.txt", "asasgdsg.md")
+	assert.Error(t, err)
+}
+
+func Test_parseCopyrightHolder(t *testing.T) {
+	assert.Equal(t, "2016 bpicode", parseCopyrightHolder("Copyright (c) 2016 bpicode"))
+	assert.Equal(t, "John Doe <[email protected]>", parseCopyrightHolder("Copyright (c) John Doe <[email protected]>"))
+	assert.Equal(t, "2009 The Go Authors. All rights reserved.", parseCopyrightHolder("Copyright (c) 2009 The Go Authors. All rights reserved."))
+	assert.Equal(t, "2011 John Doe <[email protected]>", parseCopyrightHolder("Copyright © 2011 John Doe <[email protected]>"))
+}

tools/notice2copyright/copyright_writer.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+)
+
+type copyrightWriter struct {
+	root project
+	deps []project
+}
+
+func (c *copyrightWriter) writeTo(w io.Writer) {
+	c.writeHead(w)
+	c.writeProject(w, c.root)
+	c.writeDeps(w)
+	c.writeLics(w)
+}
+
+func (c *copyrightWriter) writeHead(w io.Writer) {
+	fmt.Fprintln(w, "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/")
+	fmt.Fprintf(w, "Upstream-Name: %s\n", c.root.name)
+	fmt.Fprintf(w, "Source: %s\n", c.root.url)
+	fmt.Fprintln(w, "")
+}
+
+func (c *copyrightWriter) writeProject(w io.Writer, p project) {
+	if p.isRoot {
+		fmt.Fprintln(w, "Files: *")
+	} else {
+		fmt.Fprintf(w, "Files: %s/*\n", p.dir())
+	}
+
+	fmt.Fprintf(w, "Copyright: %s\n", p.copyrightHolders[0])
+	for _, other := range p.copyrightHolders[1:] {
+		fmt.Fprintf(w, "           %s\n", other)
+	}
+	fmt.Fprintf(w, "License: %s\n\n", p.license.shortName)
+}
+
+func (c *copyrightWriter) writeDeps(w io.Writer) {
+	sort.Slice(c.deps, func(i, j int) bool {
+		return c.deps[i].name < c.deps[j].name
+	})
+	for _, d := range c.deps {
+		c.writeProject(w, d)
+	}
+}
+
+func (c *copyrightWriter) writeLics(w io.Writer) {
+	m := c.licSet()
+	allLics := c.sortByShortName(m)
+	for _, l := range allLics {
+		c.writeLic(w, l)
+	}
+}
+
+func (c *copyrightWriter) licSet() map[license]interface{} {
+	m := make(map[license]interface{})
+	m[c.root.license] = nil
+	for _, p := range c.deps {
+		m[p.license] = nil
+	}
+	return m
+}
+
+func (c *copyrightWriter) sortByShortName(m map[license]interface{}) []license {
+	allLics := make([]license, 0)
+	for l := range m {
+		allLics = append(allLics, l)
+	}
+	sort.Slice(allLics, func(i, j int) bool {
+		return allLics[i].shortName < allLics[j].shortName
+	})
+	return allLics
+}
+
+func (c *copyrightWriter) writeLic(w io.Writer, l license) {
+	fmt.Printf("License: %s\n", l.shortName)
+	s := bufio.NewScanner(bytes.NewBufferString(l.text))
+	for s.Scan() {
+		line := s.Text()
+		if strings.TrimSpace(line) == "" {
+			fmt.Fprintln(w, " .")
+		} else {
+			fmt.Fprintf(w, " %s\n", line)
+		}
+	}
+	fmt.Fprintln(w, "")
+}