From 4c11bf38fe42f7b7805c99864ee084ec671668de Mon Sep 17 00:00:00 2001 From: Alexander Fisher Date: Fri, 10 Jan 2025 14:33:03 +0000 Subject: [PATCH] Add `remote_pql_query` function Perform a PuppetDB query on an arbitrary PuppetDB server If you need to query a PuppetDB server that is not connected to your Puppet Server, (perhaps part of a separate Puppet installation that uses its own PKI), then this function is for you! --- .sync.yml | 6 +- Gemfile | 1 + REFERENCE.md | 96 ++++++++++++++ .../functions/extlib/remote_pql_query.rb | 123 ++++++++++++++++++ .../functions/extlib/remote_pql_query_spec.rb | 104 +++++++++++++++ 5 files changed, 328 insertions(+), 2 deletions(-) create mode 100644 lib/puppet/functions/extlib/remote_pql_query.rb create mode 100644 spec/functions/extlib/remote_pql_query_spec.rb diff --git a/.sync.yml b/.sync.yml index 835ef80..b899b3f 100644 --- a/.sync.yml +++ b/.sync.yml @@ -1,3 +1,5 @@ --- -.travis.yml: - secure: "IkrfAnec7ovZLMvhvXt8ZihyYdAJTC/nm7KDm4u2G/uD2NGaMdHNOAenkwIwC1vfCzHKcgC5u/lAYFrYvHpQpJW0kHLKnk1SpndfWX9kd5SlDDzEP5mJGjMZeTY6H9sV5fsB6Pt7l/sw5ACL/0bFDl0mYBnVhGv6UxZZ5xMQIUw=" +Gemfile: + optional: + ':test': + - gem: 'puppetdb-ruby' diff --git a/Gemfile b/Gemfile index 2ac98f8..b17a86c 100644 --- a/Gemfile +++ b/Gemfile @@ -8,6 +8,7 @@ group :test do gem 'coveralls', :require => false gem 'simplecov-console', :require => false gem 'puppet_metadata', '~> 4.0', :require => false + gem 'puppetdb-ruby', :require => false end group :development do diff --git a/REFERENCE.md b/REFERENCE.md index 762fdb6..2ce3ac4 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -28,6 +28,7 @@ Thus making it directly usable with the values from facter. * [`extlib::path_join`](#extlib--path_join): Take one or more paths and join them together * [`extlib::random_password`](#extlib--random_password): A function to return a string of arbitrary length that contains randomly selected characters. * [`extlib::read_url`](#extlib--read_url): Fetch a string from a URL (should only be used with 'small' remote files). This function should only be used with trusted/internal sources. +* [`extlib::remote_pql_query`](#extlib--remote_pql_query): Perform a PuppetDB query on an arbitrary PuppetDB server If you need to query a PuppetDB server that is not connected to your Puppet Server * [`extlib::resources_deep_merge`](#extlib--resources_deep_merge): Deeply merge a "defaults" hash into a "resources" hash like the ones expected by `create_resources()`. * [`extlib::sort_by_version`](#extlib--sort_by_version): A function that sorts an array of version numbers. * [`extlib::to_ini`](#extlib--to_ini): This converts a puppet hash to an INI string. @@ -958,6 +959,101 @@ Data type: `Stdlib::HTTPUrl` The URL to read from +### `extlib::remote_pql_query` + +Type: Ruby 4.x API + +Perform a PuppetDB query on an arbitrary PuppetDB server + +If you need to query a PuppetDB server that is not connected to your Puppet +Server (perhaps part of a separate Puppet installation that uses its own +PKI), then this function is for you! + +The `puppetdb-ruby` gem _must_ be installed in your puppetserver's ruby +environment before you can use this function! + +#### `extlib::remote_pql_query(String[1] $query, HTTPSUrl $url, String[1] $key, String[1] $cert, String[1] $cacert, Optional[Hash] $options)` + +The extlib::remote_pql_query function. + +Returns: `Array` Returns the PQL query response results + +##### `query` + +Data type: `String[1]` + +The PQL query to run + +##### `url` + +Data type: `HTTPSUrl` + +The PuppetDB HTTPS URL (SSL with cert-based authentication) + +##### `key` + +Data type: `String[1]` + +The client SSL key associated with the SSL client certificate + +##### `cert` + +Data type: `String[1]` + +The client SSL cert to present to PuppetDB + +##### `cacert` + +Data type: `String[1]` + +The CA certificate + +##### `options` + +Data type: `Optional[Hash]` + +PuppetDB query options. (See https://www.puppet.com/docs/puppetdb/8/api/query/v4/paging) + +#### `extlib::remote_pql_query(String[1] $query, HTTPUrl $url, Optional[Hash] $options)` + +The extlib::remote_pql_query function. + +Returns: `Array` Returns the PQL query response results + +##### Examples + +###### 'Collecting' exported resource defined type from a foreign PuppetDB + +```puppet +$pql_results = extlib::remote_pql_query( + "resources[title,parameters] { type = \"My_Module::My_type\" and nodes { deactivated is null } and exported = true and parameters.collect_on = \"${trusted['certname']}\" }", + 'http://puppetdb.example.com:8080', +) +$pql_results.each |$result| { + my_module::my_type { $result['title']: + * => $result['parameters'] + } +} +``` + +##### `query` + +Data type: `String[1]` + +The PQL query to run + +##### `url` + +Data type: `HTTPUrl` + +The PuppetDB HTTP URL (non SSL version) + +##### `options` + +Data type: `Optional[Hash]` + +PuppetDB query options. (See https://www.puppet.com/docs/puppetdb/8/api/query/v4/paging) + ### `extlib::resources_deep_merge` Type: Ruby 4.x API diff --git a/lib/puppet/functions/extlib/remote_pql_query.rb b/lib/puppet/functions/extlib/remote_pql_query.rb new file mode 100644 index 0000000..2fe2005 --- /dev/null +++ b/lib/puppet/functions/extlib/remote_pql_query.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +require 'tempfile' + +# Perform a PuppetDB query on an arbitrary PuppetDB server +# +# If you need to query a PuppetDB server that is not connected to your Puppet +# Server (perhaps part of a separate Puppet installation that uses its own +# PKI), then this function is for you! +# +# The `puppetdb-ruby` gem _must_ be installed in your puppetserver's ruby +# environment before you can use this function! +Puppet::Functions.create_function(:'extlib::remote_pql_query') do + local_types do + type 'HTTPUrl = Pattern[/(?i:\Ahttp:\/\/.*\z)/]' + type 'HTTPSUrl = Pattern[/(?i:\Ahttps:\/\/.*\z)/]' + end + + # @param query The PQL query to run + # @param url The PuppetDB HTTPS URL (SSL with cert-based authentication) + # @param key The client SSL key associated with the SSL client certificate + # @param cert The client SSL cert to present to PuppetDB + # @param cacert The CA certificate + # @param options PuppetDB query options. (See https://www.puppet.com/docs/puppetdb/8/api/query/v4/paging) + # @return Returns the PQL query response results + dispatch :secure_remote_pql_query do + param 'String[1]', :query + param 'HTTPSUrl', :url + param 'String[1]', :key + param 'String[1]', :cert + param 'String[1]', :cacert + optional_param 'Hash', :options + return_type 'Array' + end + + # @param query The PQL query to run + # @param url The PuppetDB HTTP URL (non SSL version) + # @param options PuppetDB query options. (See https://www.puppet.com/docs/puppetdb/8/api/query/v4/paging) + # @return Returns the PQL query response results + # @example 'Collecting' exported resource defined type from a foreign PuppetDB + # $pql_results = extlib::remote_pql_query( + # "resources[title,parameters] { type = \"My_Module::My_type\" and nodes { deactivated is null } and exported = true and parameters.collect_on = \"${trusted['certname']}\" }", + # 'http://puppetdb.example.com:8080', + # ) + # $pql_results.each |$result| { + # my_module::my_type { $result['title']: + # * => $result['parameters'] + # } + # } + dispatch :insecure_remote_pql_query do + param 'String[1]', :query + param 'HTTPUrl', :url + optional_param 'Hash', :options + return_type 'Array' + end + + def secure_remote_pql_query(query, url, key, cert, cacert, options = {}) + keyfile = Tempfile.new('remote_pql_query_keyfile') + certfile = Tempfile.new('remote_pql_query_certfile') + cafile = Tempfile.new('remote_pql_query_cafile') + + begin + keyfile.write(key) + keyfile.close + + certfile.write(cert) + certfile.close + + cafile.write(cacert) + cafile.close + + client_options = { + server: url, + pem: { + 'key' => keyfile.path, + 'cert' => certfile.path, + 'ca_file' => cafile.path, + } + } + + remote_pql_query(query, options, client_options) + ensure + [keyfile, certfile, cafile].each(&:unlink) + end + end + + def insecure_remote_pql_query(query, url, options = {}) + client_options = { server: url } + + remote_pql_query(query, options, client_options) + end + + def remote_pql_query(query, query_options, client_options) + require 'puppetdb' + + # If the dalen/puppetdbquery module is installed, then there'll be a clash + # of libraries/namespaces and we need to manually require the files from + # puppetdb-ruby... + unless PuppetDB.constants.include?(:Client) + require 'puppetdb/client' + require 'puppetdb/query' + require 'puppetdb/response' + require 'puppetdb/error' + require 'puppetdb/config' + end + + client = PuppetDB::Client.new(client_options) + + begin + response = client.request( + '', # PQL + query, + query_options + ) + + response.data + rescue PuppetDB::APIError => e + raise Puppet::Error, "PuppetDB API Error: #{e.response.inspect}" + rescue StandardError => e + raise Puppet::Error, "Remote PQL query failed: #{e.message}" + end + end +end diff --git a/spec/functions/extlib/remote_pql_query_spec.rb b/spec/functions/extlib/remote_pql_query_spec.rb new file mode 100644 index 0000000..cac1ec8 --- /dev/null +++ b/spec/functions/extlib/remote_pql_query_spec.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'puppetdb' + +describe 'extlib::remote_pql_query' do + let(:mock_client) { instance_double(PuppetDB::Client) } + let(:mock_response) { PuppetDB::Response.new(['test_result']) } + + before do + allow(PuppetDB::Client).to receive(:new).and_return(mock_client) + allow(mock_client).to receive(:request).and_return(mock_response) + end + + context 'secure_remote_pql_query' do + it 'returns the data array for valid HTTPS params' do + is_expected.to run.with_params( + 'facts { name = "osfamily" }', # query + 'https://puppetdb.example.com', # URL (matches HTTPS dispatch) + 'client_key', # key + 'client_cert', # cert + 'ca_cert' # cacert + ).and_return(['test_result']) + end + + it 'raises ArgumentError if given an HTTP URL in the secure dispatch' do + is_expected.to run.with_params( + 'facts { name = "osfamily" }', + 'http://puppetdb.example.com', # Wrong for secure dispatch + 'client_key', + 'client_cert', + 'ca_cert' + ).and_raise_error( + ArgumentError, %r{parameter 'url'}i + ) + end + end + + context 'insecure_remote_pql_query' do + it 'returns the data array for valid HTTP params' do + is_expected.to run.with_params( + 'facts { name = "osfamily" }', # query + 'http://puppetdb.example.com' # URL (matches HTTP dispatch) + ).and_return(['test_result']) + end + + it 'raises ArgumentError if given an HTTPS URL in the insecure dispatch' do + is_expected.to run.with_params( + 'facts { name = "osfamily" }', + 'https://puppetdb.example.com' # Wrong for insecure dispatch + ).and_raise_error( + ArgumentError, %r{parameter 'url'}i + ) + end + end + + context 'with query options' do + it 'passes options to the client.request call' do + allow(mock_client).to receive(:request).with( + '', + 'resources { type = "File" }', + { 'limit' => 5 } + ).and_return(mock_response) + + is_expected.to run.with_params( + 'resources { type = "File" }', + 'http://puppetdb.example.com', + { 'limit' => 5 } + ).and_return(['test_result']) + + expect(mock_client).to have_received(:request).with( + '', + 'resources { type = "File" }', + { 'limit' => 5 } + ) + end + end + + context 'when PuppetDB::APIError is raised' do + it 're-raises as a Puppet::Error' do + allow(mock_client).to receive(:request).and_raise( + PuppetDB::APIError.new( + instance_double(PuppetDB::Response, inspect: 'some API error') + ) + ) + + is_expected.to run.with_params( + 'facts { name = "osfamily" }', + 'http://puppetdb.example.com' + ).and_raise_error(Puppet::Error, %r{PuppetDB API Error: some API error}) + end + end + + context 'when a generic error is raised' do + it 're-raises as a Puppet::Error' do + allow(mock_client).to receive(:request).and_raise(RuntimeError, 'boom') + + is_expected.to run.with_params( + 'facts { name = "osfamily" }', + 'http://puppetdb.example.com' + ).and_raise_error(Puppet::Error, %r{Remote PQL query failed: boom}) + end + end +end