From b73ad1099a3f91273314c32514d1e8603a82f1d3 Mon Sep 17 00:00:00 2001 From: kr172t Date: Sun, 4 Feb 2024 22:35:50 +0200 Subject: [PATCH] fix curl and nginx to handle large messages --- curl/scripts/run-curl-loop.sh | 16 +++++++++++++-- curl/src/curl/curl.service.spec.ts | 9 ++++---- curl/src/curl/curl.service.ts | 8 +++----- curl/src/utils/message.generator.spec.ts | 26 ------------------------ curl/src/utils/message.generator.ts | 9 -------- nginx/nginx.conf | 1 + 6 files changed, 22 insertions(+), 47 deletions(-) delete mode 100644 curl/src/utils/message.generator.spec.ts delete mode 100644 curl/src/utils/message.generator.ts diff --git a/curl/scripts/run-curl-loop.sh b/curl/scripts/run-curl-loop.sh index 52b590e7..41bba63b 100755 --- a/curl/scripts/run-curl-loop.sh +++ b/curl/scripts/run-curl-loop.sh @@ -5,7 +5,19 @@ nginx_host="$1" nginx_port="$2" iteration_count="$3" algorithm="$4" -payload="$5" +message_size="$5" num_processes=$(($(getconf _NPROCESSORS_ONLN) * 2)) -seq ${iteration_count} | xargs -P $num_processes -n 1 -I % curl https://${nginx_host}:${nginx_port} -k --curves ${algorithm} -XPOST -d "$payload" -H "Content-Type: text/plain" -o /dev/null \ No newline at end of file +# Generate random message with specified size +generate_message() { + local size="$1" + tr -dc '[:print:]' /tmp/message.txt +} + +# Generate the message +generate_message "$message_size" + +# Perform curl requests using xargs +#seq "$iteration_count" | xargs -P "$num_processes" -n 1 -I % sh -c "curl 'https://${nginx_host}:${nginx_port}' -k --curves '${algorithm}' -XPOST -d @/tmp/message.txt -H 'Content-Type: text/plain' -o /dev/null" +#seq ${iteration_count} | xargs -P $num_processes -n 1 -I % curl https://${nginx_host}:${nginx_port} -k --curves ${algorithm} -XPOST -d "$payload" -H "Content-Type: text/plain" -o /dev/null +seq "$iteration_count" | xargs -P "$num_processes" -n 1 -I % curl "https://${nginx_host}:${nginx_port}" -k --curves "${algorithm}" -XPOST -d "@/tmp/message.txt" -H "Content-Type: text/plain" -o /dev/null \ No newline at end of file diff --git a/curl/src/curl/curl.service.spec.ts b/curl/src/curl/curl.service.spec.ts index af771882..a3830af1 100644 --- a/curl/src/curl/curl.service.spec.ts +++ b/curl/src/curl/curl.service.spec.ts @@ -5,7 +5,6 @@ import { ConfigModule, ConfigService } from '@nestjs/config'; import { HttpException, HttpStatus } from '@nestjs/common'; import * as shellJS from 'shelljs'; -import {MessageGenerator} from "../utils/message.generator"; jest.mock('shelljs', () => ({ exec: jest.fn(), })); @@ -56,7 +55,7 @@ describe('CurlService', () => { const runCurlsSpy = jest.spyOn(curlService, 'runCurls').mockResolvedValue(undefined); await curlService.run(curlRequest); expect(validateSpy).toHaveBeenCalledWith(curlRequest); - expect(runCurlsSpy).toHaveBeenCalledWith(curlRequest.iterationsCount, curlRequest.algorithm, expect.any(String)); + expect(runCurlsSpy).toHaveBeenCalledWith(curlRequest.iterationsCount, curlRequest.algorithm, curlRequest.messageSize); }); it('should throw an HttpException with status INTERNAL_SERVER_ERROR when runCurls throws an error', async () => { @@ -124,10 +123,10 @@ describe('CurlService', () => { it('should call execAsync with the correct command', async () => { const iterationsCount = 1000; const algorithm = 'kyber512'; - const message = MessageGenerator.generate(8); + const messageSize = 1024; const execAsyncSpy = jest.spyOn(curlService, 'execAsync').mockResolvedValue(undefined); - await curlService['runCurls'](iterationsCount, algorithm, message); - const expectedCommand = curlService['format'](`./scripts/run-curl-loop.sh ${configService.get('nginx.host')} ${configService.get('nginx.port')} ${iterationsCount} ${algorithm} ${message}`); + await curlService['runCurls'](iterationsCount, algorithm, messageSize); + const expectedCommand = curlService['format'](`./scripts/run-curl-loop.sh ${configService.get('nginx.host')} ${configService.get('nginx.port')} ${iterationsCount} ${algorithm} ${messageSize}`); expect(execAsyncSpy).toHaveBeenCalledWith(expectedCommand); }); // Add more test cases for error handling in runCurls. diff --git a/curl/src/curl/curl.service.ts b/curl/src/curl/curl.service.ts index 0957a637..cd29f2e1 100644 --- a/curl/src/curl/curl.service.ts +++ b/curl/src/curl/curl.service.ts @@ -2,7 +2,6 @@ import { Injectable, HttpException, HttpStatus } from '@nestjs/common'; import * as shellJS from 'shelljs'; import { CurlRequest } from '../dto/curl-request.dto'; import { ConfigService } from '@nestjs/config'; -import {MessageGenerator} from '../utils/message.generator'; @Injectable() export class CurlService { @@ -20,8 +19,7 @@ export class CurlService { async run(curlRequest: CurlRequest): Promise { this.validate(curlRequest); try { - const message = MessageGenerator.generate(curlRequest.messageSize); - await this.runCurls(curlRequest.iterationsCount, curlRequest.algorithm, message); + await this.runCurls(curlRequest.iterationsCount, curlRequest.algorithm, curlRequest.messageSize); } catch (err) { this.processIsRunning = false; console.error('[CurlService:run] Error occurred: ', err); @@ -40,8 +38,8 @@ export class CurlService { } } - private async runCurls(iterationsCount: number, algorithm: string, message: string) { - const curlCommand = this.format(`${this.CURL_SCRIPT_PATH} ${this.configService.get('nginx.host')} ${this.configService.get('nginx.port')} ${iterationsCount} ${algorithm} ${message}`); + private async runCurls(iterationsCount: number, algorithm: string, messageSize: number) { + const curlCommand = this.format(`${this.CURL_SCRIPT_PATH} ${this.configService.get('nginx.host')} ${this.configService.get('nginx.port')} ${iterationsCount} ${algorithm} ${messageSize}`); this.processIsRunning = true; await this.execAsync(curlCommand); console.log('[CurlService:run] Finished taking all curl samples'); diff --git a/curl/src/utils/message.generator.spec.ts b/curl/src/utils/message.generator.spec.ts deleted file mode 100644 index a687fba4..00000000 --- a/curl/src/utils/message.generator.spec.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { MessageGenerator } from './message.generator'; - -describe('MessageGenerator', () => { - let service: MessageGenerator; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [MessageGenerator], - }).compile(); - - service = module.get(MessageGenerator); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); - - describe('generate', () => { - it('should generate a message with the specified size', () => { - const sizeInBytes = 10; - const generatedMessage = MessageGenerator.generate(sizeInBytes); - expect(generatedMessage.length).toBe(sizeInBytes); - }); - }); -}); diff --git a/curl/src/utils/message.generator.ts b/curl/src/utils/message.generator.ts deleted file mode 100644 index 05c19fc7..00000000 --- a/curl/src/utils/message.generator.ts +++ /dev/null @@ -1,9 +0,0 @@ -import {Injectable} from '@nestjs/common'; - -@Injectable() -export class MessageGenerator { - static generate(sizeInBytes: number) { - // Generate the string by repeating the character 'a' - return 'a'.repeat(sizeInBytes); } - -} diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 638b883a..59f33205 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -15,6 +15,7 @@ http { keepalive_timeout 65; + client_max_body_size 0; # Allow unlimited size for the entire request body server { listen 8080;