Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@Xearty
Xearty committed Oct 23, 2024
1 parent 1fc04f4 commit 5aa94c7
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 52 deletions.
6 changes: 2 additions & 4 deletions packages/mcl/src/src/mcl/commands/ci.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,9 @@ import mcl.utils.process : execute;
import mcl.utils.nix : nix;
import mcl.utils.json : toJSON;

Params params;

export void ci()
{
params = parseEnv!Params;
Params params = parseEnv!Params;

auto shardMatrix = generateShardMatrix();
foreach (shard; shardMatrix.include)
Expand Down Expand Up @@ -53,7 +51,7 @@ export void ci()
}

auto matrix = flakeAttr(params.flakePre, arch, os, params.flakePost)
.nixEvalJobs(cachixUrl, false);
.nixEvalJobs(params, cachixUrl, false);

foreach (pkg; matrix)
{
Expand Down
99 changes: 53 additions & 46 deletions packages/mcl/src/src/mcl/commands/ci_matrix.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,15 @@ import std.path : buildPath;
import std.process : pipeProcess, wait, Redirect, kill;
import std.exception : enforce;
import std.format : fmt = format;
import std.logger : tracef, infof, errorf;
import std.logger : tracef, infof, errorf, warningf;

import mcl.utils.env : optional, MissingEnvVarsException, parseEnv;
import mcl.utils.string : enumToString, StringRepresentation, MaxWidth, writeRecordAsTable;
import mcl.utils.json : toJSON;
import mcl.utils.path : rootDir, resultDir, gcRootsDir, createResultDirs;
import mcl.utils.process : execute;
import mcl.utils.nix : nix;
import mcl.utils.cachix : cachixNixStoreUrl;

enum GitHubOS
{
Expand Down Expand Up @@ -141,18 +142,12 @@ version (unittest)
];
}

immutable Params params;

version (unittest) {} else
shared static this()
{
params = parseEnv!Params;
}

export void ci_matrix()
{
Params params = parseEnv!Params;

createResultDirs();
nixEvalForAllSystems().array.printTableForCacheStatus();
nixEvalForAllSystems(params).array.printTableForCacheStatus(params);
}

string flakeAttr(string prefix, SupportedSystem system, string postfix)
Expand All @@ -167,14 +162,14 @@ string flakeAttr(string prefix, string arch, string os, string postfix)
return "%s.%s-%s%s".fmt(prefix, arch, os, postfix);
}

Package[] checkCacheStatus(Package[] packages)
Package[] checkCacheStatus(Package[] packages, string cachixAuthToken)
{
import std.array : appender;
import std.parallelism : parallel;

foreach (ref pkg; packages.parallel)
{
pkg = checkPackage(pkg);
pkg = checkPackage(pkg, cachixAuthToken);
struct Output { string isCached, name, storePath; }
auto res = appender!string;
writeRecordAsTable(
Expand All @@ -188,11 +183,13 @@ Package[] checkCacheStatus(Package[] packages)

export void print_table()
{
Params params = parseEnv!Params;

createResultDirs();

getPrecalcMatrix()
.checkCacheStatus()
.printTableForCacheStatus();
getPrecalcMatrix(params)
.checkCacheStatus(params.cachixAuthToken)
.printTableForCacheStatus(params);
}

struct Params
Expand Down Expand Up @@ -304,12 +301,12 @@ unittest
}
}

Package[] nixEvalJobs(string flakeAttrPrefix, string cachixUrl, bool doCheck = true)
Package[] nixEvalJobs(string flakeAttrPrefix, Params params, string cachixUrl, bool doCheck = true)
{
Package[] result = [];

int maxMemoryMB = getAvailableMemoryMB();
int maxWorkers = getNixEvalWorkerCount();
int maxMemoryMB = getAvailableMemoryMB(params.maxMemory);
int maxWorkers = getNixEvalWorkerCount(params.maxWorkers);

const args = [
"nix-eval-jobs", "--quiet", "--option", "warn-dirty", "false",
Expand All @@ -322,32 +319,34 @@ Package[] nixEvalJobs(string flakeAttrPrefix, string cachixUrl, bool doCheck = t

auto pipes = pipeProcess(args, Redirect.stdout | Redirect.stderr);

void logWarning(string errorMsg)
{
warningf("Command `%s` stderr:\n---\n%s\n---", args, errorMsg);
}

void logError(string errorMsg)
{
errorf("Command `%s` failed with error:\n---\n%s\n---",
args, errorMsg);
}

bool jobFailed = false;

foreach (line; pipes.stdout.byLine)
{
if (line.indexOf("{") == -1)
{
errorf("Expected JSON object on stdout from nix-eval-jobs, got: `%s`", line);
continue;
}

auto json = parseJSON(line);

if (auto err = "error" in json)
{
logError((*err).str);
jobFailed = true;
continue; // drain the output
}

Package pkg = json.packageFromNixEvalJobsJson(
flakeAttrPrefix, cachixUrl);

if (doCheck)pkg = pkg.checkPackage();
if (doCheck)pkg = pkg.checkPackage(params.cachixAuthToken);

result ~= pkg;

Expand All @@ -365,16 +364,20 @@ Package[] nixEvalJobs(string flakeAttrPrefix, string cachixUrl, bool doCheck = t
output: pkg.output
).writeRecordAsTable(stderr.lockingTextWriter);
}

string bufferedOutput = "";
foreach (line; pipes.stderr.byLine)
{
if (uselessWarnings.map!((warning) => line.indexOf(warning) != -1).any)
continue;

logError(line.idup);
bufferedOutput ~= line ~ "\n";
}

logWarning(bufferedOutput);

int status = wait(pipes.pid);
enforce(status == 0, "Command `%s` failed with status %s".fmt(args, status));
enforce(status == 0 && !jobFailed, "Command `%s` failed with status %s".fmt(args, status));

return result;
}
Expand All @@ -396,26 +399,26 @@ SupportedSystem[] getSupportedSystems(string flakeRef = ".")
return json.array.map!(system => getSystem(system.str)).array;
}

Package[] nixEvalForAllSystems()
Package[] nixEvalForAllSystems(Params params)
{
const cachixUrl = "https://" ~ params.cachixCache ~ ".cachix.org";
const cachixUrl = cachixNixStoreUrl(params.cachixCache);
const systems = getSupportedSystems();

infof("Evaluating flake for: %s", systems);

return systems.map!(system =>
flakeAttr(params.flakePre, system, params.flakePost)
.nixEvalJobs(cachixUrl)
.nixEvalJobs(params, cachixUrl)
)
.reduce!((a, b) => a ~ b)
.array
.sort!((a, b) => a.name < b.name)
.array;
}

int getNixEvalWorkerCount()
int getNixEvalWorkerCount(int maxWorkers)
{
return params.maxWorkers == 0 ? (threadsPerCPU() < 8 ? threadsPerCPU() : 8) : params.maxWorkers;
return maxWorkers == 0 ? (threadsPerCPU() < 8 ? threadsPerCPU() : 8) : maxWorkers;
}

@("getNixEvalWorkerCount")
Expand All @@ -424,7 +427,7 @@ unittest
assert(getNixEvalWorkerCount() == (threadsPerCPU() < 8 ? threadsPerCPU() : 8));
}

int getAvailableMemoryMB()
int getAvailableMemoryMB(int maxMemory)
{

// free="$(< /proc/meminfo grep MemFree | tr -s ' ' | cut -d ' ' -f 2)"
Expand All @@ -448,15 +451,15 @@ int getAvailableMemoryMB()
.find!(a => a.indexOf("Shmem:") != -1)
.front
.split[1].to!int;
int maxMemoryMB = params.maxMemory == 0 ? ((free + cached + buffers + shmem) / 1024)
: params.maxMemory;
int maxMemoryMB = maxMemory == 0 ? ((free + cached + buffers + shmem) / 1024) : maxMemory;
return maxMemoryMB;
}

@("getAvailableMemoryMB")
unittest
{
assert(getAvailableMemoryMB() > 0);
Params params = parseEnv!Params;
assert(getAvailableMemoryMB(params.maxMemory) > 0);
}

void saveCachixDeploySpec(Package[] packages)
Expand All @@ -481,12 +484,12 @@ unittest
assert(testPackageArray[1].output == deploySpec[0]["out"].str);
}

void saveGHCIMatrix(Package[] packages)
void saveGHCIMatrix(Package[] packages, bool isInitial)
{
auto matrix = JSONValue([
"include": JSONValue(packages.map!(pkg => pkg.toJSON()).array)
]);
string resPath = rootDir.buildPath(params.isInitial ? "matrix-pre.json" : "matrix-post.json");
string resPath = rootDir.buildPath(isInitial ? "matrix-pre.json" : "matrix-post.json");
resPath.write(JSONValue(matrix).toString(JSONOptions.doNotEscapeSlashes));
}

Expand All @@ -495,8 +498,10 @@ unittest
{
import std.file : rmdirRecurse;

Params params = parseEnv!Params;

createResultDirs();
saveGHCIMatrix(cast(Package[]) testPackageArray);
saveGHCIMatrix(cast(Package[]) testPackageArray, params.isInitial);
JSONValue matrix = rootDir
.buildPath(params.isInitial ? "matrix-pre.json" : "matrix-post.json")
.readText
Expand Down Expand Up @@ -624,24 +629,24 @@ unittest

}

void printTableForCacheStatus(Package[] packages)
void printTableForCacheStatus(Package[] packages, Params params)
{
if (params.precalcMatrix == "")
{
saveGHCIMatrix(packages);
saveGHCIMatrix(packages, params.isInitial);
}
saveCachixDeploySpec(packages);
saveGHCIComment(convertNixEvalToTableSummary(packages, params.isInitial));
}

Package checkPackage(Package pkg)
Package checkPackage(Package pkg, ref string cachixAuthToken)
{
import std.algorithm : canFind;
import std.string : lineSplitter;
import std.net.curl : HTTP, httpGet = get, HTTPStatusException;

auto http = HTTP();
http.addRequestHeader("Authorization", "Bearer " ~ params.cachixAuthToken);
http.addRequestHeader("Authorization", "Bearer " ~ cachixAuthToken);

try
{
Expand All @@ -663,6 +668,8 @@ Package checkPackage(Package pkg)
@("checkPackage")
unittest
{
Params params = parseEnv!Params;

const nixosCacheEndpoint = "https://cache.nixos.org/";
const storePathHash = "mdb034kf7sq6g03ric56jxr4a7043l41";
const storePath = "/nix/store/" ~ storePathHash ~ "-hello-2.12.1";
Expand All @@ -673,14 +680,14 @@ unittest
);

assert(!testPackage.isCached);
assert(checkPackage(testPackage).isCached);
assert(checkPackage(testPackage, params.cachixAuthToken).isCached);

testPackage.cacheUrl = nixosCacheEndpoint ~ "nonexistent.narinfo";

assert(!checkPackage(testPackage).isCached);
assert(!checkPackage(testPackage, params.cachixAuthToken).isCached);
}

Package[] getPrecalcMatrix()
Package[] getPrecalcMatrix(Params params)
{
auto precalcMatrixStr = params.precalcMatrix == "" ? "{\"include\": []}" : params.precalcMatrix;
enforce!MissingEnvVarsException(
Expand Down
7 changes: 5 additions & 2 deletions packages/mcl/src/src/mcl/commands/deploy_spec.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,19 @@ import mcl.utils.cachix : cachixNixStoreUrl, DeploySpec, createMachineDeploySpec
import mcl.utils.tui : bold;
import mcl.utils.json : toJSON, fromJSON, tryDeserializeFromJsonFile;

import mcl.commands.ci_matrix : flakeAttr, params, Params, nixEvalJobs, SupportedSystem;
import mcl.commands.ci_matrix : flakeAttr, Params, nixEvalJobs, SupportedSystem;

export void deploy_spec()
{

const deploySpecFile = resultDir.buildPath("cachix-deploy-spec.json");

if (!exists(deploySpecFile))
{
Params params = parseEnv!Params;

auto nixosConfigs = flakeAttr("legacyPackages", SupportedSystem.x86_64_linux, "bareMetalMachines")
.nixEvalJobs(params.cachixCache.cachixNixStoreUrl);
.nixEvalJobs(params, params.cachixCache.cachixNixStoreUrl);

auto configsMissingFromCachix = nixosConfigs.filter!(c => !c.isCached);

Expand Down

0 comments on commit 5aa94c7

Please sign in to comment.