fix/homebrew uses from macos (teaxyz#15)

* remove uses from macos

* remove trailing commas

* remove from the query

* use import_id, not name

package_managers/homebrew/README.md

@@ -0,0 +1,16 @@
+# Homebrew
+
+## Notes
+
+- Homebrew's dependencies are not just restricted to the `{build,test,...}_dependencies`
+  fields listed in the JSON APIs...it also uses some system level packages denoted in
+  `uses_from_macos`, and `variations` (for linux). The pipeline currently does consider
+  these dependencies.
+- Homebrew's JSON API and formula.rb files do not specify all the versions available for
+  a package. It does provide the `stable` and `head` versions, which are pulled in
+  [`versions.jq`](jq/versions.jq).
+- Versioned formulae (like `python`, `postgresql`) are ones where the Homebrew package
+  specifies a version. The pipeline considers these packages individual packages,
+  and so creates new records in the `packages` table.
+- The data source for Homebrew does not retrieve the analytics information that is
+  available via the individual JSON API endpoints for each package.

package_managers/homebrew/jq/dependencies.jq

@@ -1,26 +1,24 @@
-# TODO: variations (linux only, by architecture)
+# TODO: variations (linux only, by architecture), uses_from_macos
 
 [.[] | {
   package_name: .name,
   build_deps: .build_dependencies,
   runtime_deps: .dependencies,
   recommended_deps: .recommended_dependencies,
   test_deps: .test_dependencies,
-  optional_deps: .optional_dependencies,
-  uses_from_macos: .uses_from_macos
+  optional_deps: .optional_dependencies
 } | 
   # here's where we'd substitute the depends_on_type ids, for each depends_on type ids
   # the `[]` at the end is to ensure that we're exploding the arrays, so each dependency gets its own row!
     {package_name: .package_name, depends_on_type: $build_deps_type_id, depends_on: .build_deps[]},
     {package_name: .package_name, depends_on_type: $runtime_deps_type_id, depends_on: .runtime_deps[]},
     {package_name: .package_name, depends_on_type: $recommended_deps_type_id, depends_on: .recommended_deps[]},
     {package_name: .package_name, depends_on_type: $test_deps_type_id, depends_on: .test_deps[]},
-    {package_name: .package_name, depends_on_type: $optional_deps_type_id, depends_on: .optional_deps[]},
-    {package_name: .package_name, depends_on_type: $uses_from_macos_type_id, depends_on: .uses_from_macos[]}
+    {package_name: .package_name, depends_on_type: $optional_deps_type_id, depends_on: .optional_deps[]}
   |
   # now, filter out the null dependencies
   select(.depends_on != null) |
-  # and only look at the ones that are strings TODO: some are JSONs?
+  # and only look at the ones that are strings (some objects are present)
   select(.depends_on | type == "string") | 
   # generate the sql statements!
   "INSERT INTO dependencies (version_id, dependency_id, dependency_type_id) VALUES (

package_managers/homebrew/jq/package_url.jq

@@ -13,7 +13,7 @@
   |
   # and here we say "for each url, generate an insert statement"
   "INSERT INTO package_urls (package_id, url_id) VALUES (
-    (SELECT id FROM packages WHERE name = '" + .package_name + "'),
+    (SELECT id FROM packages WHERE import_id = '" + .package_name + "'),
     (SELECT id FROM urls WHERE url = '" + .url + "' AND url_type_id = '" + .type + "'))
     ON CONFLICT DO NOTHING;"
 ] | join("\n")

package_managers/homebrew/pipeline.sh

@@ -30,8 +30,7 @@ IFS='|' read -r \
     RUNTIME_DEPENDS_ON_TYPE_ID \
     RECOMMENDED_DEPENDS_ON_TYPE_ID \
     OPTIONAL_DEPENDS_ON_TYPE_ID \
-    TEST_DEPENDS_ON_TYPE_ID \
-    USES_FROM_MACOS_DEPENDS_ON_TYPE_ID <<< "$IDS"
+    TEST_DEPENDS_ON_TYPE_ID <<< "$IDS"
 
 # Validate that all required IDs are present and export them
 required_vars=(
@@ -43,7 +42,6 @@ required_vars=(
     RECOMMENDED_DEPENDS_ON_TYPE_ID
     OPTIONAL_DEPENDS_ON_TYPE_ID
     TEST_DEPENDS_ON_TYPE_ID
-    USES_FROM_MACOS_DEPENDS_ON_TYPE_ID
 )
 
 for var in "${required_vars[@]}"; do
@@ -104,7 +102,6 @@ if [ "$FETCH" = true ]; then
                     --arg recommended_deps_type_id "$RECOMMENDED_DEPENDS_ON_TYPE_ID" \
                     --arg optional_deps_type_id "$OPTIONAL_DEPENDS_ON_TYPE_ID" \
                     --arg test_deps_type_id "$TEST_DEPENDS_ON_TYPE_ID" \
-                    --arg uses_from_macos_type_id "$USES_FROM_MACOS_DEPENDS_ON_TYPE_ID" \
                     "$DATA_DIR"/latest/source.json > "$DATA_DIR"/latest/"${filename}".sql
                 ;;
             *)

package_managers/homebrew/sql/homebrew_vars.sql

@@ -16,10 +16,7 @@ SELECT
     ) AS optional_depends_on_type_id,
     (
         SELECT id FROM depends_on_types WHERE name = 'test'
-    ) AS test_depends_on_type_id,
-    (
-        SELECT id FROM depends_on_types WHERE name = 'uses_from_macos'
-    ) AS uses_from_macos_depends_on_type_id
+    ) AS test_depends_on_type_id
 FROM
     package_managers AS pm
 INNER JOIN