Unified Queue: implement correct fleet-initiated flag and setup exper…

…ience priority (#25448)

ee/server/service/setup_experience.go

@@ -178,8 +178,10 @@ func (svc *Service) SetupExperienceNextStep(ctx context.Context, hostUUID string
 	case len(installersPending) > 0:
 		// enqueue installers
 		for _, installer := range installersPending {
-			// TODO(mna): this should be top priority as this is setup exp.
-			installUUID, err := svc.ds.InsertSoftwareInstallRequest(ctx, host.ID, *installer.SoftwareInstallerID, false, nil)
+			installUUID, err := svc.ds.InsertSoftwareInstallRequest(ctx, host.ID, *installer.SoftwareInstallerID, fleet.HostSoftwareInstallOptions{
+				SelfService:        false,
+				ForSetupExperience: true,
+			})
 			if err != nil {
 				return false, ctxerr.Wrap(ctx, err, "queueing setup experience install request")
 			}
@@ -208,8 +210,10 @@ func (svc *Service) SetupExperienceNextStep(ctx context.Context, hostUUID string
 				},
 			}
 
-			// TODO(mna): setup experience must be higher-priority
-			cmdUUID, err := svc.installSoftwareFromVPP(ctx, host, vppApp, true, false)
+			cmdUUID, err := svc.installSoftwareFromVPP(ctx, host, vppApp, true, fleet.HostSoftwareInstallOptions{
+				SelfService:        false,
+				ForSetupExperience: true,
+			})
 			if err != nil {
 				return false, ctxerr.Wrap(ctx, err, "queueing vpp app installation")
 			}

ee/server/service/setup_experience_test.go

@@ -56,7 +56,7 @@ func TestSetupExperienceNextStep(t *testing.T) {
 		return mockListHostsLite, nil
 	}
 
-	ds.InsertSoftwareInstallRequestFunc = func(ctx context.Context, hostID, softwareInstallerID uint, selfService bool, policyID *uint) (string, error) {
+	ds.InsertSoftwareInstallRequestFunc = func(ctx context.Context, hostID, softwareInstallerID uint, opts fleet.HostSoftwareInstallOptions) (string, error) {
 		requestedInstalls[hostID] = append(requestedInstalls[hostID], softwareInstallerID)
 		return "install-uuid", nil
 	}

ee/server/service/software_installers.go

@@ -16,7 +16,6 @@ import (
 
 	"github.com/fleetdm/fleet/v4/pkg/file"
 	"github.com/fleetdm/fleet/v4/pkg/fleethttp"
-	"github.com/fleetdm/fleet/v4/server/authz"
 	authz_ctx "github.com/fleetdm/fleet/v4/server/contexts/authz"
 	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
 	hostctx "github.com/fleetdm/fleet/v4/server/contexts/host"
@@ -1001,17 +1000,19 @@ func (svc *Service) InstallSoftwareTitle(ctx context.Context, hostID uint, softw
 		return ctxerr.Wrap(ctx, err, "finding VPP app for title")
 	}
 
-	_, err = svc.installSoftwareFromVPP(ctx, host, vppApp, mobileAppleDevice || fleet.AppleDevicePlatform(platform) == fleet.MacOSPlatform, false)
+	_, err = svc.installSoftwareFromVPP(ctx, host, vppApp, mobileAppleDevice || fleet.AppleDevicePlatform(platform) == fleet.MacOSPlatform, fleet.HostSoftwareInstallOptions{
+		SelfService: false,
+	})
 	return err
 }
 
-func (svc *Service) installSoftwareFromVPP(ctx context.Context, host *fleet.Host, vppApp *fleet.VPPApp, appleDevice bool, selfService bool) (string, error) {
+func (svc *Service) installSoftwareFromVPP(ctx context.Context, host *fleet.Host, vppApp *fleet.VPPApp, appleDevice bool, opts fleet.HostSoftwareInstallOptions) (string, error) {
 	token, err := svc.GetVPPTokenIfCanInstallVPPApps(ctx, appleDevice, host)
 	if err != nil {
 		return "", err
 	}
 
-	return svc.InstallVPPAppPostValidation(ctx, host, vppApp, token, selfService, nil)
+	return svc.InstallVPPAppPostValidation(ctx, host, vppApp, token, opts)
 }
 
 func (svc *Service) GetVPPTokenIfCanInstallVPPApps(ctx context.Context, appleDevice bool, host *fleet.Host) (string, error) {
@@ -1057,7 +1058,7 @@ func (svc *Service) GetVPPTokenIfCanInstallVPPApps(ctx context.Context, appleDev
 	return token, nil
 }
 
-func (svc *Service) InstallVPPAppPostValidation(ctx context.Context, host *fleet.Host, vppApp *fleet.VPPApp, token string, selfService bool, policyID *uint) (string, error) {
+func (svc *Service) InstallVPPAppPostValidation(ctx context.Context, host *fleet.Host, vppApp *fleet.VPPApp, token string, opts fleet.HostSoftwareInstallOptions) (string, error) {
 	// at this moment, neither the UI nor the back-end are prepared to
 	// handle [asyncronous errors][1] on assignment, so before assigning a
 	// device to a license, we need to:
@@ -1133,7 +1134,7 @@ func (svc *Service) InstallVPPAppPostValidation(ctx context.Context, host *fleet
 
 	// enqueue the VPP app command to install
 	cmdUUID := uuid.NewString()
-	err = svc.ds.InsertHostVPPSoftwareInstall(ctx, host.ID, vppApp.VPPAppID, cmdUUID, eventID, selfService, policyID)
+	err = svc.ds.InsertHostVPPSoftwareInstall(ctx, host.ID, vppApp.VPPAppID, cmdUUID, eventID, opts)
 	if err != nil {
 		return "", ctxerr.Wrapf(ctx, err, "inserting host vpp software install for host with serial %s and app with adamID %s", host.HardwareSerial, vppApp.AdamID)
 	}
@@ -1159,7 +1160,9 @@ func (svc *Service) installSoftwareTitleUsingInstaller(ctx context.Context, host
 		}
 	}
 
-	_, err := svc.ds.InsertSoftwareInstallRequest(ctx, host.ID, installer.InstallerID, false, nil)
+	_, err := svc.ds.InsertSoftwareInstallRequest(ctx, host.ID, installer.InstallerID, fleet.HostSoftwareInstallOptions{
+		SelfService: false,
+	})
 	return ctxerr.Wrap(ctx, err, "inserting software install request")
 }
 
@@ -1249,40 +1252,45 @@ func (svc *Service) UninstallSoftwareTitle(ctx context.Context, hostID uint, sof
 		}
 	}
 
-	// Get the uninstall script and use the standard script infrastructure to run it.
-	contents, err := svc.ds.GetAnyScriptContents(ctx, installer.UninstallScriptContentID)
-	if err != nil {
-		if fleet.IsNotFound(err) {
-			return ctxerr.Wrap(ctx,
-				fleet.NewInvalidArgumentError("software_title_id", `No uninstall script exists for the provided "software_title_id".`).
-					WithStatus(http.StatusNotFound), "getting uninstall script contents")
-		}
-		return err
-	}
-
-	var teamID uint
-	if host.TeamID != nil {
-		teamID = *host.TeamID
-	}
-	// create the script execution request; the host will be notified of the
-	// script execution request via the orbit config's Notifications mechanism.
-	request := fleet.HostScriptRequestPayload{
-		HostID:          host.ID,
-		ScriptContents:  string(contents),
-		ScriptContentID: installer.UninstallScriptContentID,
-		TeamID:          teamID,
-	}
-	if ctxUser := authz.UserFromContext(ctx); ctxUser != nil {
-		request.UserID = &ctxUser.ID
-	}
-	scriptResult, err := svc.ds.NewInternalScriptExecutionRequest(ctx, &request)
-	if err != nil {
-		return ctxerr.Wrap(ctx, err, "create script execution request")
-	}
+	// TODO(mna): an uninstall request will first go in the upcoming_activities
+	// queue, and only when that activity is ready to run will the internal script
+	// be created in host_script_requests. Keeping it here but commented, for when
+	// we implement the queue execution.
+
+	// // Get the uninstall script and use the standard script infrastructure to run it.
+	// contents, err := svc.ds.GetAnyScriptContents(ctx, installer.UninstallScriptContentID)
+	// if err != nil {
+	// 	if fleet.IsNotFound(err) {
+	// 		return ctxerr.Wrap(ctx,
+	// 			fleet.NewInvalidArgumentError("software_title_id", `No uninstall script exists for the provided "software_title_id".`).
+	// 				WithStatus(http.StatusNotFound), "getting uninstall script contents")
+	// 	}
+	// 	return err
+	// }
+	// var teamID uint
+	// if host.TeamID != nil {
+	// 	teamID = *host.TeamID
+	// }
+	// // create the script execution request; the host will be notified of the
+	// // script execution request via the orbit config's Notifications mechanism.
+	// request := fleet.HostScriptRequestPayload{
+	// 	HostID:          host.ID,
+	// 	ScriptContents:  string(contents),
+	// 	ScriptContentID: installer.UninstallScriptContentID,
+	// 	TeamID:          teamID,
+	// }
+	// if ctxUser := authz.UserFromContext(ctx); ctxUser != nil {
+	// 	request.UserID = &ctxUser.ID
+	// }
+	// scriptResult, err := svc.ds.NewInternalScriptExecutionRequest(ctx, &request)
+	// if err != nil {
+	// 	return ctxerr.Wrap(ctx, err, "create script execution request")
+	// }
 
 	// Update the host software installs table with the uninstall request.
 	// Pending uninstalls will automatically show up in the UI Host Details -> Activity -> Upcoming tab.
-	if err = svc.insertSoftwareUninstallRequest(ctx, scriptResult.ExecutionID, host, installer); err != nil {
+	execID := uuid.NewString()
+	if err = svc.insertSoftwareUninstallRequest(ctx, execID, host, installer); err != nil {
 		return err
 	}
 
@@ -1830,7 +1838,9 @@ func (svc *Service) SelfServiceInstallSoftwareTitle(ctx context.Context, host *f
 			}
 		}
 
-		_, err = svc.ds.InsertSoftwareInstallRequest(ctx, host.ID, installer.InstallerID, true, nil)
+		_, err = svc.ds.InsertSoftwareInstallRequest(ctx, host.ID, installer.InstallerID, fleet.HostSoftwareInstallOptions{
+			SelfService: true,
+		})
 		return ctxerr.Wrap(ctx, err, "inserting self-service software install request")
 	}
 
@@ -1864,7 +1874,9 @@ func (svc *Service) SelfServiceInstallSoftwareTitle(ctx context.Context, host *f
 	platform := host.FleetPlatform()
 	mobileAppleDevice := fleet.AppleDevicePlatform(platform) == fleet.IOSPlatform || fleet.AppleDevicePlatform(platform) == fleet.IPadOSPlatform
 
-	_, err = svc.installSoftwareFromVPP(ctx, host, vppApp, mobileAppleDevice || fleet.AppleDevicePlatform(platform) == fleet.MacOSPlatform, true)
+	_, err = svc.installSoftwareFromVPP(ctx, host, vppApp, mobileAppleDevice || fleet.AppleDevicePlatform(platform) == fleet.MacOSPlatform, fleet.HostSoftwareInstallOptions{
+		SelfService: true,
+	})
 	return err
 }
 

ee/server/service/software_installers_test.go

@@ -105,7 +105,7 @@ func TestInstallUninstallAuth(t *testing.T) {
 	ds.GetHostLastInstallDataFunc = func(ctx context.Context, hostID uint, installerID uint) (*fleet.HostLastInstallData, error) {
 		return nil, nil
 	}
-	ds.InsertSoftwareInstallRequestFunc = func(ctx context.Context, hostID uint, softwareInstallerID uint, selfService bool, policyID *uint) (string,
+	ds.InsertSoftwareInstallRequestFunc = func(ctx context.Context, hostID uint, softwareInstallerID uint, opts fleet.HostSoftwareInstallOptions) (string,
 		error,
 	) {
 		return "request_id", nil

server/datastore/mysql/activities.go

@@ -266,10 +266,9 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint
 
 	listStmts := []string{
 		// list pending scripts
-		// TODO(mna): should the user name IF use fleet_initiated?
 		`SELECT
 			ua.execution_id as uuid,
-			IF(sua.policy_id IS NOT NULL, 'Fleet', COALESCE(u.name, JSON_EXTRACT(ua.payload, '$.user.name'))) as name,
+			IF(ua.fleet_initiated, 'Fleet', COALESCE(u.name, JSON_EXTRACT(ua.payload, '$.user.name'))) as name,
 			u.id as user_id,
 			COALESCE(u.gravatar_url, JSON_EXTRACT(ua.payload, '$.user.gravatar_url')) as gravatar_url,
 			COALESCE(u.email, JSON_EXTRACT(ua.payload, '$.user.email')) as user_email,
@@ -307,12 +306,9 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint
 			ua.activity_type = 'script'
 `,
 		// list pending software installs
-		// TODO(mna): should the user name IF use fleet_initiated?
 		`SELECT
 			ua.execution_id as uuid,
-			-- policies with automatic installers generate a host_software_installs with (user_id=NULL,self_service=0),
-			-- so we mark those as "Fleet"
-			IF(ua.user_id IS NULL AND NOT JSON_EXTRACT(ua.payload, '$.self_service'), 'Fleet', COALESCE(u.name, JSON_EXTRACT(ua.payload, '$.user.name'))) AS name,
+			IF(ua.fleet_initiated, 'Fleet', COALESCE(u.name, JSON_EXTRACT(ua.payload, '$.user.name'))) AS name,
 			ua.user_id as user_id,
 			COALESCE(u.gravatar_url, JSON_EXTRACT(ua.payload, '$.user.gravatar_url')) as gravatar_url,
 			COALESCE(u.email, JSON_EXTRACT(ua.payload, '$.user.email')) as user_email,
@@ -352,12 +348,9 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint
 			ua.activity_type = 'software_install'
 		`,
 		// list pending software uninstalls
-		// TODO(mna): should the user name IF use fleet_initiated?
 		`SELECT
 			ua.execution_id as uuid,
-			-- policies with automatic installers generate a host_software_installs with (user_id=NULL,self_service=0),
-			-- so we mark those as "Fleet"
-			IF(ua.user_id IS NULL, 'Fleet', COALESCE(u.name, JSON_EXTRACT(ua.payload, '$.user.name'))) AS name,
+			IF(ua.fleet_initiated, 'Fleet', COALESCE(u.name, JSON_EXTRACT(ua.payload, '$.user.name'))) AS name,
 			ua.user_id as user_id,
 			COALESCE(u.gravatar_url, JSON_EXTRACT(ua.payload, '$.user.gravatar_url')) as gravatar_url,
 			COALESCE(u.email, JSON_EXTRACT(ua.payload, '$.user.email')) as user_email,

server/datastore/mysql/activities_test.go

@@ -457,7 +457,7 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) {
 
 	// install the VPP app on h1
 	// commander, _ := createMDMAppleCommanderAndStorage(t, ds)
-	err = ds.InsertHostVPPSoftwareInstall(ctx, h1.ID, vppApp.VPPAppID, vppCommand1, "event-id-1", false, nil)
+	err = ds.InsertHostVPPSoftwareInstall(ctx, h1.ID, vppApp.VPPAppID, vppCommand1, "event-id-1", fleet.HostSoftwareInstallOptions{})
 	require.NoError(t, err)
 	// err = commander.EnqueueCommand(
 	// 	ctx,
@@ -466,7 +466,7 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) {
 	// )
 	// require.NoError(t, err)
 	// install the VPP app on h2, self-service
-	err = ds.InsertHostVPPSoftwareInstall(noUserCtx, h2.ID, vppApp.VPPAppID, vppCommand2, "event-id-2", true, nil)
+	err = ds.InsertHostVPPSoftwareInstall(noUserCtx, h2.ID, vppApp.VPPAppID, vppCommand2, "event-id-2", fleet.HostSoftwareInstallOptions{SelfService: true})
 	require.NoError(t, err)
 	// err = commander.EnqueueCommand(
 	// 	ctx,
@@ -516,7 +516,7 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) {
 	// create some software installs requests for h1, make some complete
 	// h1FooFailed, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.InstallerID, false, nil)
 	// require.NoError(t, err)
-	h1Bar, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw2Meta.InstallerID, false, nil)
+	h1Bar, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw2Meta.InstallerID, fleet.HostSoftwareInstallOptions{})
 	require.NoError(t, err)
 	t.Log("h1Bar", h1Bar)
 	// err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{
@@ -541,7 +541,7 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) {
 		Query: "SELECT 1",
 	})
 	require.NoError(t, err)
-	h1Fleet, err := ds.InsertSoftwareInstallRequest(noUserCtx, h1.ID, sw1Meta.InstallerID, false, &policy.ID)
+	h1Fleet, err := ds.InsertSoftwareInstallRequest(noUserCtx, h1.ID, sw1Meta.InstallerID, fleet.HostSoftwareInstallOptions{PolicyID: &policy.ID})
 	t.Log("h1Fleet", h1Fleet)
 	require.NoError(t, err)
 
@@ -557,11 +557,11 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) {
 	// require.NoError(t, err)
 	// h2F := hsr.ExecutionID
 	// add a pending software install request for h2
-	h2Bar, err := ds.InsertSoftwareInstallRequest(ctx, h2.ID, sw2Meta.InstallerID, false, nil)
+	h2Bar, err := ds.InsertSoftwareInstallRequest(ctx, h2.ID, sw2Meta.InstallerID, fleet.HostSoftwareInstallOptions{})
 	require.NoError(t, err)
 	t.Log("h2Bar", h2Bar)
 	// No user for this one and Self-service, means it was installed by the end user, so the user_id should be null/nil.
-	h2SelfService, err := ds.InsertSoftwareInstallRequest(noUserCtx, h2.ID, sw1Meta.InstallerID, true, nil)
+	h2SelfService, err := ds.InsertSoftwareInstallRequest(noUserCtx, h2.ID, sw1Meta.InstallerID, fleet.HostSoftwareInstallOptions{SelfService: true})
 	require.NoError(t, err)
 	t.Log("h2SelfService", h2SelfService)
 
@@ -576,7 +576,7 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) {
 	t.Log("h2SetupExp", h2SetupExp)
 
 	// create pending install and uninstall requests for h3 that will be deleted
-	_, err = ds.InsertSoftwareInstallRequest(ctx, h3.ID, sw3Meta.InstallerID, false, nil)
+	_, err = ds.InsertSoftwareInstallRequest(ctx, h3.ID, sw3Meta.InstallerID, fleet.HostSoftwareInstallOptions{})
 	require.NoError(t, err)
 	err = ds.InsertSoftwareUninstallRequest(ctx, "uninstallRun", h3.ID, sw3Meta.InstallerID)
 	require.NoError(t, err)
@@ -764,8 +764,7 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) {
 					require.Equal(t, wantUser.Email, *a.ActorEmail, "result %d", i)
 				} else {
 					require.Nil(t, a.ActorID, "result %d", i)
-					// TODO(mna): this should probably become consistent across activity types, all based on fleet_initiated
-					if a.Type == (fleet.ActivityInstalledAppStoreApp{}.ActivityName()) {
+					if a.FleetInitiated {
 						require.NotNil(t, a.ActorFullName, "result %d", i)
 						require.Equal(t, "Fleet", *a.ActorFullName, "result %d", i)
 					} else {

server/datastore/mysql/hosts_test.go

@@ -7030,7 +7030,7 @@ func testHostsDeleteHosts(t *testing.T, ds *Datastore) {
 		ValidatedLabels: &fleet.LabelIdentsWithScope{},
 	})
 	require.NoError(t, err)
-	_, err = ds.InsertSoftwareInstallRequest(context.Background(), host.ID, softwareInstaller, false, nil)
+	_, err = ds.InsertSoftwareInstallRequest(context.Background(), host.ID, softwareInstaller, fleet.HostSoftwareInstallOptions{})
 	require.NoError(t, err)
 
 	// Add an awaiting configuration entry

server/datastore/mysql/policies_test.go

@@ -308,7 +308,7 @@ func testGlobalPolicyPendingScriptsAndInstalls(t *testing.T, ds *Datastore) {
 	require.Len(t, policies, 1)
 
 	// create a pending software install request
-	_, err = ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID, false, &policy2.ID)
+	_, err = ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID, fleet.HostSoftwareInstallOptions{PolicyID: &policy2.ID})
 	require.NoError(t, err)
 
 	pendingInstalls, err := ds.ListPendingSoftwareInstalls(ctx, host2.ID)
@@ -900,7 +900,7 @@ func testTeamPolicyPendingScriptsAndInstalls(t *testing.T, ds *Datastore) {
 	})
 	require.NoError(t, err)
 	// create a pending software install request
-	_, err = ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID, false, &policy2.ID)
+	_, err = ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID, fleet.HostSoftwareInstallOptions{PolicyID: &policy2.ID})
 	require.NoError(t, err)
 
 	pendingInstalls, err := ds.ListPendingSoftwareInstalls(ctx, host2.ID)

server/datastore/mysql/scripts.go

@@ -48,6 +48,7 @@ func (ds *Datastore) NewHostScriptExecutionRequest(ctx context.Context, request
 	})
 }
 
+// TODO(mna): might become unused after unified queue implementation
 func (ds *Datastore) NewInternalScriptExecutionRequest(ctx context.Context, request *fleet.HostScriptRequestPayload) (*fleet.HostScriptResult, error) {
 	var res *fleet.HostScriptResult
 	var err error
@@ -109,7 +110,7 @@ WHERE
 		request.HostID,
 		priority,
 		request.UserID,
-		isInternal || request.UserID == nil, // TODO(mna): confirm if this makes sense
+		request.PolicyID != nil, // fleet-initiated if request is via a policy failure
 		execID,
 		request.SyncRequest,
 		isInternal,