Merge pull request #23 from pmonks/issue-3

.github/ISSUE_TEMPLATE/Invalid_id_constructed.md

@@ -0,0 +1,13 @@
+---
+name: 🐛 Invalid SPDX identifier constructed
+about: When the library constructs an invalid SPDX identifier. 😢
+
+---
+
+## `lice-comb` API(s) you were calling, if known:
+
+_e.g. `lice-comb.deps/deps-licenses`_
+
+## Input data that you provided to that API:
+
+_e.g. a license name, or the URI of a file containing the license text, or the `tools.deps` coordinate of the dependency, etc._

.github/workflows/ci.yml

@@ -17,12 +17,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
           java-version: 17
-      - uses: DeLaGuardo/setup-clojure@11.0
+      - uses: DeLaGuardo/setup-clojure@12.1
         with:
           cli: latest
       - uses: actions/cache@v3

.github/workflows/dependencies.yml

@@ -11,5 +11,5 @@ jobs:
     container:
       image: uochan/antq
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - run: java -jar /tmp/antq/antq.jar --skip=pom --error-format="::error file={{file}}::{{message}}"

.github/workflows/deploy.yml

@@ -10,14 +10,14 @@ jobs:
     environment: clojars
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0    # Make sure we get the full history, or else the version number gets screwed up
       - uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
           java-version: 17
-      - uses: DeLaGuardo/setup-clojure@11.0
+      - uses: DeLaGuardo/setup-clojure@12.1
         with:
           cli: latest
       - uses: actions/cache@v3

.github/workflows/docs.yml

@@ -9,12 +9,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
           java-version: 17
-      - uses: DeLaGuardo/setup-clojure@11.0
+      - uses: DeLaGuardo/setup-clojure@12.1
         with:
           cli: latest
       - uses: actions/cache@v3

.gitignore

@@ -1,4 +1,5 @@
 .idea
+.nvd
 *.iml
 pom.xml
 pom.xml.asc

NOTICE

@@ -0,0 +1,4 @@
+lice-comb
+Copyright © 2021 Peter Monks (https://github.com/pmonks)
+
+This project contains source code for rdrop-while, which is copyright Joshua Suskalo (https://github.com/IGJoshua) 2023 and licensed as "CC0-1.0 OR MIT". For details, see https://discord.com/channels/729136623421227082/732641743723298877/1141786961875583097.

README.md

@@ -9,36 +9,134 @@
 
 # lice-comb
 
-A Clojure library for software license detection.  It does this by combing through text, files, and even entire directory structures, and attempting to detect what license(s) they contain.
+A Clojure library for software *lice*nse detection.  It does this by *comb*ing through tools.deps and Leiningen dependencies, directory structures, and JAR & ZIP files, attempting to detect what license(s) they contain, and then normalising them into [SPDX license expression(s)](https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/).
 
 This library leverages, and is inspired by, the *excellent* [SPDX project](https://spdx.dev/).  It's a great shame that it doesn't have greater traction in the Java & Clojure (and wider open source) communities.  If you're new to SPDX and would prefer to read a primer rather than dry specification documents, I can thoroughly recommend [David A. Wheeler's SPDX Tutorial](https://github.com/david-a-wheeler/spdx-tutorial#spdx-tutorial).
 
-## Using the library
+## System Requirements
 
-### Documentation
+* `lice-comb` (all versions) requires an internet connection.
 
-[API documentation is available here](https://pmonks.github.io/lice-comb/).
+* `lice-comb` (all versions) assumes Maven is installed and in the `PATH` (but has fallback logic if it isn't available).
 
-[An FAQ is available here](https://github.com/pmonks/lice-comb/wiki/FAQ).
+* `lice-comb` (v2.0+) requires JDK 11 or higher.
+
+## Installation
+
+`lice-comb` is available as a Maven artifact from [Clojars](https://clojars.org/com.github.pmonks/lice-comb).
+
+### Trying it Out
+
+#### Clojure CLI
 
-### Dependency
+```shell
+$ # Where #.#.# is replaced with an actual version number (see badge above)
+$ clj -Sdeps '{:deps {com.github.pmonks/lice-comb {:mvn/version "#.#.#"}}}'
+```
+
+#### Leiningen
+
+```shell
+$ lein try com.github.pmonks/lice-comb
+```
 
-Express the correct maven dependencies in your `deps.edn`:
+#### deps-try
 
-```edn
-{:deps {com.github.pmonks/lice-comb {:mvn/version "LATEST_CLOJARS_VERSION"}}}
+```shell
+$ deps-try com.github.pmonks/lice-comb
 ```
 
-### Require one or more of the namespaces
+### Demo
 
 ```clojure
-(ns your.ns
-  (:require [lice-comb.deps  :as lcd]
-            [lice-comb.files :as lcf]
-            [lice-comb.maven :as lcm]
-            [lice-comb.spdx  :as lcs]))
+;; License name, uri and full text matching
+(require '[lice-comb.matching :as lcm])
+
+; Initialise the matching namespace
+; Notes:
+; 1. This is slow (takes ~1 minute on my laptop), almost all of which is Spdx-Java-Library's initialisation (see https://github.com/spdx/Spdx-Java-Library/issues/193)
+; 2. This step is optional, though initialisation will still happen regardless, and when it does you'll incur the same cost
+(lcm/init!)
+
+(lcm/name->expressions "Apache")
+;=> #{"Apache-2.0"}
+
+(lcm/name->expressions "GNU Public License 2.0 w/ the GNU Classpath Exception")
+;=> #{"GPL-2.0-only WITH Classpath-exception-2.0"}
+
+(lcm/text->ids (slurp "https://www.apache.org/licenses/LICENSE-2.0.txt"))
+;=> #{"Apache-2.0"}
+
+(lcm/uri->ids "https://www.apache.org/licenses/LICENSE-2.0.txt")
+;=> #{"Apache-2.0"}
+
+;; License extraction from Maven poms, including ones that aren't locally downloaded
+(require '[lice-comb.maven :as lcmvn])
+
+(lcmvn/pom->expressions (str (System/getProperty "user.home") "/.m2/repository/org/clojure/clojure/1.11.1/clojure-1.11.1.pom"))
+;=> #{"EPL-1.0"}
+
+(lcmvn/pom->expressions "https://repo1.maven.org/maven2/org/springframework/spring-core/6.0.11/spring-core-6.0.11.pom")
+;=> #{"Apache-2.0"}
+
+;; License extraction from tools.deps dependency maps
+(require '[lice-comb.deps :as lcd])
+
+(lcd/dep->expressions ['org.clojure/clojure {:deps/manifest :mvn :mvn/version "1.11.1"}])
+;=> #{"EPL-1.0"}
+
+;; Information about matches (useful for better understanding how lice-comb arrived at a given set of expressions, and
+;; how confident it is in the values it's providing)
+(lcm/name->expressions-info "Apache-2.0")
+;=> {"Apache-2.0" ({:type :declared, :strategy :spdx-expression, :source ("Apache-2.0")})}
+
+(lcm/name->expressions-info "GNU Public License 2.0 or later w/ the GNU Classpath Exception")
+;=> {"GPL-2.0-or-later WITH Classpath-exception-2.0"
+;     ({:type :concluded, :confidence :low, :strategy :expression-inference, :source ("GNU Public License 2.0 or later w/ the GNU Classpath Exception")}
+;      {:id "GPL-2.0-or-later", :type :concluded, :confidence :medium, :strategy :regex-matching, :source ("GNU Public License 2.0 or later w/ the GNU Classpath Exception"
+;                                                                                                          "GNU Public License 2.0 or later")}
+;      {:id "Classpath-exception-2.0", :type :concluded, :confidence :low, :strategy :regex-matching, :source ("GNU Public License 2.0 or later w/ the GNU Classpath Exception"
+;                                                                                                              "the GNU Classpath Exception"
+;                                                                                                              "Classpath Exception")})}
+
+(lcmvn/pom->expressions-info "https://repo.clojars.org/canvas/canvas/0.1.6/canvas-0.1.6.pom")
+;=> {"EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0"
+;     ({:type :declared, :strategy :spdx-expression, :source ("https://repo.clojars.org/canvas/canvas/0.1.6/canvas-0.1.6.pom"
+;                                                             "<name>"
+;                                                             "EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0")})}
+
+;; Pretty print expressions-info
+(require '[lice-comb.utils :as lcu])
+
+(println (lcu/expressions-info->string (lcd/dep->expressions-info ['com.amazonaws/aws-java-sdk-s3 {:deps/manifest :mvn :mvn/version "1.12.129"}])))
+;=> Apache-2.0:
+;     Concluded
+;       Confidence: high
+;       Strategy: regular expression matching
+;       Source:
+;       > com.amazonaws/[email protected]
+;       > https://repo.maven.apache.org/maven2/com/amazonaws/aws-java-sdk-s3/1.12.129/aws-java-sdk-s3-1.12.129.pom
+;       > https://repo.maven.apache.org/maven2/com/amazonaws/aws-java-sdk-pom/1.12.129/aws-java-sdk-pom-1.12.129.pom
+;       > <name>
+;       > Apache License, Version 2.0
+nil
 ```
 
+### API Documentation
+
+[API documentation is available here](https://pmonks.github.io/lice-comb/), or [here on cljdoc](https://cljdoc.org/d/com.github.pmonks/lice-comb/).
+
+[An FAQ is available here](https://github.com/pmonks/lice-comb/wiki/FAQ).
+
+## Upgrading
+
+### 1.x -> 2.x
+
+The implementation of [issue #3](https://github.com/pmonks/lice-comb/issues/3) resulted in a number of unavoidable breaking changes, including:
+
+* A wholesale change from returning sets of SPDX identifiers to returning sets of SPDX expressions
+* The creation of [a dedicated SPDX-specific library (`clj-spdx`)](https://github.com/pmonks/clj-spdx) that leverages [the official SPDX Java library](https://github.com/spdx/Spdx-Java-Library)
+
 ## Contributor Information
 
 [Contributor FAQ](https://github.com/pmonks/lice-comb/wiki/FAQ#contributor-faqs)

deps.edn

@@ -19,12 +19,16 @@
 {:paths ["src" "resources"]
  :deps
    {org.clojure/tools.logging             {:mvn/version "1.2.4"}
+    commons-validator/commons-validator   {:mvn/version "1.7"}
     org.clojure/data.xml                  {:mvn/version "0.2.0-alpha8"}
-    cheshire/cheshire                     {:mvn/version "5.11.0"}
     clj-xml-validation/clj-xml-validation {:mvn/version "1.0.2"}
-    camel-snake-kebab/camel-snake-kebab   {:mvn/version "0.4.3"}
-    tolitius/xml-in                       {:mvn/version "0.1.1"}}
+    tolitius/xml-in                       {:mvn/version "0.1.1"}
+    hato/hato                             {:mvn/version "0.9.0"}
+    dev.weavejester/medley                {:mvn/version "1.7.0"}
+    dom-top/dom-top                       {:mvn/version "1.0.8"}
+    miikka/clj-base62                     {:mvn/version "0.1.1"}
+    com.github.pmonks/clj-spdx            {:mvn/version "1.0.95"}
+    com.github.pmonks/rencg               {:mvn/version "1.0.34"}}
  :aliases
-   {:build {:deps       {io.github.clojure/tools.build {:git/tag "v0.9.4" :git/sha "76b78fe"}
-                         com.github.pmonks/pbr         {:mvn/version "RELEASE"}}
+   {:build {:deps       {com.github.pmonks/pbr {:mvn/version "RELEASE"}}
             :ns-default pbr.build}}}

pbr.clj

@@ -19,7 +19,7 @@
 (def lib 'com.github.pmonks/lice-comb)
 
 #_{:clj-kondo/ignore [:unresolved-namespace]}
-(def version (format "1.0.%s" (b/git-count-revs nil)))
+(def version (format "2.0.%s" (b/git-count-revs nil)))
 
 (defn set-opts
   [opts]

resources/lice_comb/names.edn

@@ -0,0 +1,8 @@
+; Map of name values seen in the wild that are too ambiguous / cursed to support any reasonable form of automated parsing
+{
+  ; Seen in https://repo.maven.apache.org/maven2/com/sun/mail/all/1.4.7/all-1.4.7.pom
+  "GPLv2+CE" {"GPL-2.0-only WITH Classpath-exception-2.0"
+              ({:type :concluded :confidence :high :strategy :manual-verification :source ("GPLv2+CE")}
+               {:id "GPL-2.0-only"            :type :concluded :confidence :high :strategy :manual-verification :source ("GPLv2+CE" "GPLv2")}
+               {:id "Classpath-exception-2.0" :type :concluded :confidence :high :strategy :manual-verification :source ("GPLv2+CE" "CE")})}
+}