refactor: blockchain artifact int models (#1616)

warehouse/dbt/models/intermediate/directory/int_contracts.sql → warehouse/dbt/models/intermediate/blockchain_artifacts/int_contracts.sql

@@ -1,3 +1,7 @@
+{#
+  TODO: This needs to be refactored or deprecated.
+#}
+
 with deployers as (
   select
     *,

warehouse/dbt/models/intermediate/blockchain_artifacts/int_deployers.sql

@@ -0,0 +1,40 @@
+{{
+  config(
+    materialized='table'
+  )
+}}
+
+{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora"] %}
+
+{% set union_queries = [] %}
+
+{% for network in networks %}
+  {% set table_name = "stg_" ~ network ~ "__deployers" %}
+  {% set network_upper = network.upper() %}
+
+  {% set query %}
+  select
+    block_timestamp,
+    transaction_hash,
+    deployer_address,
+    contract_address,
+    '{{ network_upper }}' as network,
+  from {{ ref(table_name) }}
+  {% endset %}
+
+  {% do union_queries.append(query) %}
+{% endfor %}
+
+{% set final_query = union_queries | join(' union all ') %}
+
+with deployers as (
+  {{ final_query }}
+)
+
+select
+  block_timestamp,
+  transaction_hash,
+  deployer_address,
+  contract_address,
+  network
+from deployers

warehouse/dbt/models/intermediate/blockchain_artifacts/int_derived_contracts.sql

@@ -0,0 +1,32 @@
+with factories_and_deployers as (
+  select
+    factories.block_timestamp,
+    factories.transaction_hash,
+    factories.network,
+    factories.originating_address as deployer_address,
+    deployers.deployer_address as factory_deployer_address,
+    factories.contract_address as contract_address
+  from {{ ref("int_factories") }} as factories
+  inner join {{ ref("int_deployers") }} as deployers
+    on
+      factories.factory_address = deployers.contract_address
+      and factories.network = deployers.network
+  union all
+  select
+    block_timestamp,
+    transaction_hash,
+    network,
+    deployer_address,
+    null as factory_deployer_address,
+    contract_address
+  from {{ ref("int_deployers") }}
+)
+
+select
+  block_timestamp,
+  transaction_hash,
+  network,
+  deployer_address,
+  factory_deployer_address,
+  contract_address
+from factories_and_deployers

warehouse/dbt/models/intermediate/blockchain_artifacts/int_factories.sql

@@ -0,0 +1,44 @@
+{{
+  config(
+    materialized='table'
+  )
+}}
+
+{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora"] %}
+
+{% set union_queries = [] %}
+
+{% for network in networks %}
+  {% set table_name = "stg_" ~ network ~ "__factories" %}
+  {% set network_upper = network.upper() %}
+
+  {% set query %}
+  select
+    block_timestamp,
+    transaction_hash,
+    originating_address,
+    originating_contract,
+    factory_address,
+    contract_address,
+    '{{ network_upper }}' as network,
+  from {{ ref(table_name) }}
+  {% endset %}
+
+  {% do union_queries.append(query) %}
+{% endfor %}
+
+{% set final_query = union_queries | join(' union all ') %}
+
+with factories as (
+  {{ final_query }}
+)
+
+select
+  block_timestamp,
+  transaction_hash,
+  originating_address,
+  originating_contract,
+  factory_address,
+  contract_address,
+  network
+from factories

warehouse/dbt/models/intermediate/directory/int_artifacts_in_ossd_by_project.sql

@@ -0,0 +1,141 @@
+with projects as (
+  select
+    project_id,
+    github,
+    npm,
+    blockchain
+  from {{ ref('stg_ossd__current_projects') }}
+),
+
+github_repos as (
+  select
+    "GITHUB" as artifact_source,
+    "REPOSITORY" as artifact_type,
+    projects.project_id,
+    repos.owner as artifact_namespace,
+    repos.name as artifact_name,
+    repos.url as artifact_url,
+    CAST(repos.id as STRING) as artifact_source_id
+  from projects
+  cross join
+    UNNEST(JSON_QUERY_ARRAY(projects.github)) as github
+  inner join
+    {{ ref('stg_ossd__current_repositories') }} as repos
+    on
+      LOWER(CONCAT("https://github.com/", repos.owner))
+      = LOWER(JSON_VALUE(github.url))
+      or LOWER(repos.url) = LOWER(JSON_VALUE(github.url))
+),
+
+all_npm_raw as (
+  select
+    "NPM" as artifact_source,
+    "PACKAGE" as artifact_type,
+    projects.project_id,
+    JSON_VALUE(npm.url) as artifact_source_id,
+    case
+      when
+        JSON_VALUE(npm.url) like "https://npmjs.com/package/%"
+        then SUBSTR(JSON_VALUE(npm.url), 28)
+      when
+        JSON_VALUE(npm.url) like "https://www.npmjs.com/package/%"
+        then SUBSTR(JSON_VALUE(npm.url), 31)
+    end as artifact_name,
+    JSON_VALUE(npm.url) as artifact_url
+  from projects
+  cross join
+    UNNEST(JSON_QUERY_ARRAY(projects.npm)) as npm
+),
+
+all_npm as (
+  select
+    project_id,
+    artifact_source_id,
+    artifact_source,
+    artifact_type,
+    artifact_name,
+    artifact_url,
+    SPLIT(REPLACE(artifact_name, "@", ""), "/")[SAFE_OFFSET(0)]
+      as artifact_namespace
+  from all_npm_raw
+),
+
+ossd_blockchain as (
+  select
+    projects.project_id,
+    tag as artifact_type,
+    network as artifact_namespace,
+    network as artifact_source,
+    JSON_VALUE(blockchains.address) as artifact_source_id,
+    JSON_VALUE(blockchains.address) as artifact_name,
+    JSON_VALUE(blockchains.address) as artifact_url
+  from projects
+  cross join
+    UNNEST(JSON_QUERY_ARRAY(projects.blockchain)) as blockchains
+  cross join
+    UNNEST(JSON_VALUE_ARRAY(blockchains.networks)) as network
+  cross join
+    UNNEST(JSON_VALUE_ARRAY(blockchains.tags)) as tag
+),
+
+all_artifacts as (
+  select
+    project_id,
+    artifact_source_id,
+    artifact_source,
+    artifact_type,
+    artifact_namespace,
+    artifact_name,
+    artifact_url
+  from
+    github_repos
+  union all
+  select
+    project_id,
+    artifact_source_id,
+    artifact_source,
+    artifact_type,
+    artifact_namespace,
+    artifact_name,
+    artifact_url
+  from
+    ossd_blockchain
+  union all
+  select
+    project_id,
+    artifact_source_id,
+    artifact_source,
+    artifact_type,
+    artifact_namespace,
+    artifact_name,
+    artifact_url
+  from
+    all_npm
+),
+
+all_normalized_artifacts as (
+  select distinct
+    project_id,
+    LOWER(artifact_source_id) as artifact_source_id,
+    {# 
+      artifact_source and artifact_type are considered internal constants hence
+      we apply an UPPER transform
+    #}
+    UPPER(artifact_source) as artifact_source,
+    UPPER(artifact_type) as artifact_type,
+    LOWER(artifact_namespace) as artifact_namespace,
+    LOWER(artifact_name) as artifact_name,
+    LOWER(artifact_url) as artifact_url
+  from all_artifacts
+)
+
+select
+  project_id,
+  {{ oso_id("artifact_source", "artifact_source_id") }} as `artifact_id`,
+  artifact_source_id,
+  artifact_source,
+  artifact_namespace,
+  artifact_name,
+  artifact_url,
+  artifact_type
+from all_normalized_artifacts

warehouse/dbt/models/marts/superchain/verification/rf4_contracts_by_app.sql

@@ -1,27 +1,9 @@
-{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora"] %}
-
-{% set union_factory_queries = [] %}
-
-{% for network in networks %}
-
-  {% set network_upper = network.upper() %}
-
-  {% set factory_table = "stg_" ~ network ~ "__factories" %}
-  {% set query %}
+with factories as (
   select
     factory_address,
     contract_address,
-    '{{ network_upper }}' as network
-  from {{ ref(factory_table) }}
-  {% endset %}
-  {% do union_factory_queries.append(query) %}
-
-{% endfor %}
-
-{% set all_factories = union_factory_queries | join(' union all ') %}
-
-with factories as (
-  {{ all_factories }}
+    network
+  from {{ ref('int_factories') }}
 ),
 
 app_contracts as (