From e7dffd72e2a17ace4150ab708b84baa4c94133c3 Mon Sep 17 00:00:00 2001 From: Stefan Topfstedt Date: Fri, 27 Dec 2024 16:19:10 -0800 Subject: [PATCH] implements batch processing on filtering parameters. --- classes/ilios.php | 88 ++++++++++++++++++++++++++++++++------------ tests/ilios_test.php | 73 ++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 23 deletions(-) diff --git a/classes/ilios.php b/classes/ilios.php index 77ead53..38fb768 100644 --- a/classes/ilios.php +++ b/classes/ilios.php @@ -45,6 +45,11 @@ class ilios { */ const API_BASE_PATH = '/api/v3/'; + /** + * @var int The maximum number of filtering request parameters in a given API request. + */ + const MAX_FILTER_PARAMS = 250; + /** * @var string The API access token. */ @@ -317,11 +322,10 @@ public function get_instructor_ids_from_learner_group(int $groupid): array { return array_values($instructorids); } - /** * Sends a GET request to a given API endpoint with given options. * - * @param string $path The target path fragment of the API request URL. May include query parameters. + * @param string $path The target path fragment of the API request URL. * @param string $key The name of the property that holds the requested data points in the payload. * @param array $filterby An associative array of filter options. * @param array $sortby An associative array of sort options. @@ -338,43 +342,30 @@ public function get( $this->validate_access_token($this->accesstoken); $options = ['headers' => ['X-JWT-Authorization' => 'Token ' . $this->accesstoken]]; + $url = $this->apibaseurl . $path; + // Construct query params from given filters and sort orders. // Unfortunately, http_build_query() doesn't cut it here, so we have to hand-roll this. - $queryparams = []; + $filterparams = []; if (!empty($filterby)) { foreach ($filterby as $param => $value) { if (is_array($value)) { foreach ($value as $val) { - $queryparams[] = "filters[$param][]=$val"; + $filterparams[] = "filters[$param][]=$val"; } } else { - $queryparams[] = "filters[$param]=$value"; + $filterparams[] = "filters[$param]=$value"; } } } - + $sortparams = []; if (!empty($sortby)) { foreach ($sortby as $param => $value) { - $queryparams[] = "order_by[$param]=$value"; + $sortparams[] = "order_by[$param]=$value"; } } - $url = $this->apibaseurl . $path; - - if (!empty($queryparams)) { - $url .= '?' . implode('&', $queryparams); - } - - $response = $this->httpclient->get($url, $options); - $rhett = $this->parse_result($response->getBody()); - if (!property_exists($rhett, $key)) { - throw new moodle_exception( - 'errorresponseentitynotfound', - 'enrol_ilios', - a: $key, - ); - } - return $rhett->$key; + return $this->send_get_request($url, $key, $options, $filterparams, $sortparams); } /** @@ -422,6 +413,57 @@ public function get_by_id(string $path, string $key, int $id, bool $returnnullon } } + /** + * Internal helper method for sending a GET request to the Ilios API. + * + * @param string $url The API request URL. May include query parameters. + * @param string $key The name of the property that holds the requested data points in the payload. + * @param array $options API Client options, such as header values. + * @param array $filterparams An associative array of filtering request parameters. + * @param array $sortparams An associative array of sorting request parameters. + * @return array The data points from the decoded payload. + * @throws GuzzleException + * @throws moodle_exception + */ + protected function send_get_request( + string $url, + string $key, + array $options = [], + array $filterparams = [], + array $sortparams = [] + ): array { + // Batch processing comes first. + // If the number of request parameters exceeds the given limit, + // we must chunk them up into smaller lists and run then individually, followed by results aggregation. + // This is done by recursively calling this method with chunked filtering options. + if (count($filterparams) > self::MAX_FILTER_PARAMS) { + $batches = array_chunk($filterparams, self::MAX_FILTER_PARAMS); + $results = []; + foreach ($batches as $batch) { + $result = $this->send_get_request($url, $key, $options, $batch, $sortparams); + $results = array_merge($results, $result); + } + return $results; + } + + $queryparams = array_merge($filterparams, $sortparams); + + if (!empty($queryparams)) { + $url .= '?' . implode('&', $queryparams); + } + + $response = $this->httpclient->get($url, $options); + $rhett = $this->parse_result($response->getBody()); + if (!property_exists($rhett, $key)) { + throw new moodle_exception( + 'errorresponseentitynotfound', + 'enrol_ilios', + a: $key, + ); + } + return $rhett->$key; + } + /** * Decodes and returns the given JSON-encoded input. * diff --git a/tests/ilios_test.php b/tests/ilios_test.php index e106fa2..f27284f 100644 --- a/tests/ilios_test.php +++ b/tests/ilios_test.php @@ -1094,6 +1094,79 @@ public static function get_with_filtering_and_sorting_provider(): array { ]; } + /** + * Tests get() in batch mode. + * + * @return void + * @throws GuzzleException + * @throws moodle_exception + */ + public function test_get_in_batch_mode(): void { + $this->resetAfterTest(); + $accesstoken = helper::create_valid_ilios_api_access_token(); + set_config('apikey', $accesstoken, 'enrol_ilios'); + set_config('host_url', 'http://ilios.demo', 'enrol_ilios'); + $handlerstack = HandlerStack::create(new MockHandler([ + new Response(200, [], json_encode([ + 'schools' => [ + ['id' => 1, 'title' => 'Medicine'], + ['id' => 2, 'title' => 'Pharmacy'], + ], + ])), + new Response(200, [], json_encode([ + 'schools' => [ + ['id' => 3, 'title' => 'Dentistry'], + ['id' => 4, 'title' => 'Nursing'], + ], + ])), + new Response(200, [], json_encode([ + 'schools' => [ + ['id' => 5, 'title' => 'Other'], + ], + ])), + ])); + $container = []; + $history = Middleware::history($container); + $handlerstack->push($history); + di::set(http_client::class, new http_client(['handler' => $handlerstack])); + + $ilios = di::get(ilios::class); + $filterby = ['id' => range(1000, 1564)]; + $sortby = ['name' => 'DESC']; + $data = $ilios->get('schools', 'schools', $filterby, $sortby); + + $this->assertCount(3, $container); + $this->assertEquals('GET', $container[0]['request']->getMethod()); + $this->assertEquals('ilios.demo', $container[0]['request']->getUri()->getHost()); + $this->assertEquals('/api/v3/schools', $container[0]['request']->getUri()->getPath()); + $paramsbatch1 = explode('&', urldecode($container[0]['request']->getUri()->getQuery())); + $this->assertCount(251, $paramsbatch1); // That's 250 filtering params and 1 sorting param. + $this->assertEquals('filters[id][]=1000', $paramsbatch1[0]); + $this->assertEquals('filters[id][]=1249', $paramsbatch1[249]); + $this->assertEquals('order_by[name]=DESC', $paramsbatch1[250]); + $paramsbatch2 = explode('&', urldecode($container[1]['request']->getUri()->getQuery())); + $this->assertCount(251, $paramsbatch2); + $this->assertEquals('filters[id][]=1250', $paramsbatch2[0]); + $this->assertEquals('filters[id][]=1499', $paramsbatch2[249]); + $this->assertEquals('order_by[name]=DESC', $paramsbatch2[250]); + $paramsbatch3 = explode('&', urldecode($container[2]['request']->getUri()->getQuery())); + $this->assertCount(66, $paramsbatch3); + $this->assertEquals('filters[id][]=1500', $paramsbatch3[0]); + $this->assertEquals('filters[id][]=1564', $paramsbatch3[64]); + $this->assertEquals('order_by[name]=DESC', $paramsbatch3[65]); + + $this->assertCount(5, $data); + $this->assertEquals(1, $data[0]->id); + $this->assertEquals('Medicine', $data[0]->title); + $this->assertEquals(2, $data[1]->id); + $this->assertEquals('Pharmacy', $data[1]->title); + $this->assertEquals(3, $data[2]->id); + $this->assertEquals('Dentistry', $data[2]->title); + $this->assertEquals(4, $data[3]->id); + $this->assertEquals('Nursing', $data[3]->title); + $this->assertEquals(5, $data[4]->id); + $this->assertEquals('Other', $data[4]->title); + } /** * Tests retrieving a resource by its ID from the Ilios API.