diff --git a/changes/23913-upcoming-activities-handle-scripts b/changes/23913-upcoming-activities-handle-scripts new file mode 100644 index 000000000000..2e4f31c02f68 --- /dev/null +++ b/changes/23913-upcoming-activities-handle-scripts @@ -0,0 +1,5 @@ +* Added script execution to the new `upcoming_activities` table. +* Added software installs to the new `upcoming_activities` table. +* Added vpp apps installs to the new `upcoming_activities` table. +* Updated the list upcoming activities endpoint to use the new `upcoming_activities` table as source of truth. +* Added support to activate the next activity when one is enqueued or when one is completed. diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index d7cfe52ba682..ad82cb364730 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -511,6 +511,7 @@ the way that the Fleet server works. } else { mdmPushService = nanomdm_pushsvc.New(mdmStorage, mdmStorage, pushProviderFactory, nanoMDMLogger) } + mds.WithPusher(mdmPushService) checkMDMAssets := func(names []fleet.MDMAssetName) (bool, error) { _, err = ds.GetAllMDMConfigAssetsByName(context.Background(), names, nil) diff --git a/ee/server/service/setup_experience.go b/ee/server/service/setup_experience.go index c11b1ac05f92..508597497fa4 100644 --- a/ee/server/service/setup_experience.go +++ b/ee/server/service/setup_experience.go @@ -178,7 +178,10 @@ func (svc *Service) SetupExperienceNextStep(ctx context.Context, hostUUID string case len(installersPending) > 0: // enqueue installers for _, installer := range installersPending { - installUUID, err := svc.ds.InsertSoftwareInstallRequest(ctx, host.ID, *installer.SoftwareInstallerID, false, nil) + installUUID, err := svc.ds.InsertSoftwareInstallRequest(ctx, host.ID, *installer.SoftwareInstallerID, fleet.HostSoftwareInstallOptions{ + SelfService: false, + ForSetupExperience: true, + }) if err != nil { return false, ctxerr.Wrap(ctx, err, "queueing setup experience install request") } @@ -207,7 +210,10 @@ func (svc *Service) SetupExperienceNextStep(ctx context.Context, hostUUID string }, } - cmdUUID, err := svc.installSoftwareFromVPP(ctx, host, vppApp, true, false) + cmdUUID, err := svc.installSoftwareFromVPP(ctx, host, vppApp, true, fleet.HostSoftwareInstallOptions{ + SelfService: false, + ForSetupExperience: true, + }) if err != nil { return false, ctxerr.Wrap(ctx, err, "queueing vpp app installation") } @@ -224,9 +230,12 @@ func (svc *Service) SetupExperienceNextStep(ctx context.Context, hostUUID string return false, ctxerr.Errorf(ctx, "setup experience script missing content id: %d", *script.SetupExperienceScriptID) } req := &fleet.HostScriptRequestPayload{ - HostID: host.ID, - ScriptName: script.Name, - ScriptContentID: *script.ScriptContentID, + HostID: host.ID, + ScriptName: script.Name, + ScriptContentID: *script.ScriptContentID, + // because the script execution request is associated with setup experience, + // it will be enqueued with a higher priority and will run before other + // items in the queue. SetupExperienceScriptID: script.SetupExperienceScriptID, } res, err := svc.ds.NewHostScriptExecutionRequest(ctx, req) diff --git a/ee/server/service/setup_experience_test.go b/ee/server/service/setup_experience_test.go index 444851c49e2b..290b52912023 100644 --- a/ee/server/service/setup_experience_test.go +++ b/ee/server/service/setup_experience_test.go @@ -56,7 +56,7 @@ func TestSetupExperienceNextStep(t *testing.T) { return mockListHostsLite, nil } - ds.InsertSoftwareInstallRequestFunc = func(ctx context.Context, hostID, softwareInstallerID uint, selfService bool, policyID *uint) (string, error) { + ds.InsertSoftwareInstallRequestFunc = func(ctx context.Context, hostID, softwareInstallerID uint, opts fleet.HostSoftwareInstallOptions) (string, error) { requestedInstalls[hostID] = append(requestedInstalls[hostID], softwareInstallerID) return "install-uuid", nil } diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index 02794196d037..753a73da5c02 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -16,7 +16,6 @@ import ( "github.com/fleetdm/fleet/v4/pkg/file" "github.com/fleetdm/fleet/v4/pkg/fleethttp" - "github.com/fleetdm/fleet/v4/server/authz" authz_ctx "github.com/fleetdm/fleet/v4/server/contexts/authz" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" hostctx "github.com/fleetdm/fleet/v4/server/contexts/host" @@ -1001,17 +1000,19 @@ func (svc *Service) InstallSoftwareTitle(ctx context.Context, hostID uint, softw return ctxerr.Wrap(ctx, err, "finding VPP app for title") } - _, err = svc.installSoftwareFromVPP(ctx, host, vppApp, mobileAppleDevice || fleet.AppleDevicePlatform(platform) == fleet.MacOSPlatform, false) + _, err = svc.installSoftwareFromVPP(ctx, host, vppApp, mobileAppleDevice || fleet.AppleDevicePlatform(platform) == fleet.MacOSPlatform, fleet.HostSoftwareInstallOptions{ + SelfService: false, + }) return err } -func (svc *Service) installSoftwareFromVPP(ctx context.Context, host *fleet.Host, vppApp *fleet.VPPApp, appleDevice bool, selfService bool) (string, error) { +func (svc *Service) installSoftwareFromVPP(ctx context.Context, host *fleet.Host, vppApp *fleet.VPPApp, appleDevice bool, opts fleet.HostSoftwareInstallOptions) (string, error) { token, err := svc.GetVPPTokenIfCanInstallVPPApps(ctx, appleDevice, host) if err != nil { return "", err } - return svc.InstallVPPAppPostValidation(ctx, host, vppApp, token, selfService, nil) + return svc.InstallVPPAppPostValidation(ctx, host, vppApp, token, opts) } func (svc *Service) GetVPPTokenIfCanInstallVPPApps(ctx context.Context, appleDevice bool, host *fleet.Host) (string, error) { @@ -1057,7 +1058,7 @@ func (svc *Service) GetVPPTokenIfCanInstallVPPApps(ctx context.Context, appleDev return token, nil } -func (svc *Service) InstallVPPAppPostValidation(ctx context.Context, host *fleet.Host, vppApp *fleet.VPPApp, token string, selfService bool, policyID *uint) (string, error) { +func (svc *Service) InstallVPPAppPostValidation(ctx context.Context, host *fleet.Host, vppApp *fleet.VPPApp, token string, opts fleet.HostSoftwareInstallOptions) (string, error) { // at this moment, neither the UI nor the back-end are prepared to // handle [asyncronous errors][1] on assignment, so before assigning a // device to a license, we need to: @@ -1121,14 +1122,19 @@ func (svc *Service) InstallVPPAppPostValidation(ctx context.Context, host *fleet } } - // add command to install - cmdUUID := uuid.NewString() - err = svc.mdmAppleCommander.InstallApplication(ctx, []string{host.UUID}, cmdUUID, vppApp.AdamID) - if err != nil { - return "", ctxerr.Wrapf(ctx, err, "sending command to install VPP %s application to host with serial %s", vppApp.AdamID, host.HardwareSerial) - } + // TODO(mna): should we associate the device (give the license) only when the + // upcoming activity is ready to run? I don't think so, because then it could + // fail when it's ready to run which is probably a worse UX as once enqueued + // you expect it to succeed. But eventually, we should do better management + // of the licenses, e.g. if the upcoming activity gets cancelled, it should + // release the reserved license. + // + // But the command is definitely not enqueued now, only when activating the + // activity. - err = svc.ds.InsertHostVPPSoftwareInstall(ctx, host.ID, vppApp.VPPAppID, cmdUUID, eventID, selfService, policyID) + // enqueue the VPP app command to install + cmdUUID := uuid.NewString() + err = svc.ds.InsertHostVPPSoftwareInstall(ctx, host.ID, vppApp.VPPAppID, cmdUUID, eventID, opts) if err != nil { return "", ctxerr.Wrapf(ctx, err, "inserting host vpp software install for host with serial %s and app with adamID %s", host.HardwareSerial, vppApp.AdamID) } @@ -1154,7 +1160,9 @@ func (svc *Service) installSoftwareTitleUsingInstaller(ctx context.Context, host } } - _, err := svc.ds.InsertSoftwareInstallRequest(ctx, host.ID, installer.InstallerID, false, nil) + _, err := svc.ds.InsertSoftwareInstallRequest(ctx, host.ID, installer.InstallerID, fleet.HostSoftwareInstallOptions{ + SelfService: false, + }) return ctxerr.Wrap(ctx, err, "inserting software install request") } @@ -1244,8 +1252,9 @@ func (svc *Service) UninstallSoftwareTitle(ctx context.Context, hostID uint, sof } } - // Get the uninstall script and use the standard script infrastructure to run it. - contents, err := svc.ds.GetAnyScriptContents(ctx, installer.UninstallScriptContentID) + // Get the uninstall script to validate there is one, will use the standard + // script infrastructure to run it. + _, err = svc.ds.GetAnyScriptContents(ctx, installer.UninstallScriptContentID) if err != nil { if fleet.IsNotFound(err) { return ctxerr.Wrap(ctx, @@ -1255,32 +1264,11 @@ func (svc *Service) UninstallSoftwareTitle(ctx context.Context, hostID uint, sof return err } - var teamID uint - if host.TeamID != nil { - teamID = *host.TeamID - } - // create the script execution request; the host will be notified of the - // script execution request via the orbit config's Notifications mechanism. - request := fleet.HostScriptRequestPayload{ - HostID: host.ID, - ScriptContents: string(contents), - ScriptContentID: installer.UninstallScriptContentID, - TeamID: teamID, - } - if ctxUser := authz.UserFromContext(ctx); ctxUser != nil { - request.UserID = &ctxUser.ID - } - scriptResult, err := svc.ds.NewInternalScriptExecutionRequest(ctx, &request) - if err != nil { - return ctxerr.Wrap(ctx, err, "create script execution request") - } - - // Update the host software installs table with the uninstall request. // Pending uninstalls will automatically show up in the UI Host Details -> Activity -> Upcoming tab. - if err = svc.insertSoftwareUninstallRequest(ctx, scriptResult.ExecutionID, host, installer); err != nil { + execID := uuid.NewString() + if err = svc.insertSoftwareUninstallRequest(ctx, execID, host, installer); err != nil { return err } - return nil } @@ -1818,7 +1806,9 @@ func (svc *Service) SelfServiceInstallSoftwareTitle(ctx context.Context, host *f } } - _, err = svc.ds.InsertSoftwareInstallRequest(ctx, host.ID, installer.InstallerID, true, nil) + _, err = svc.ds.InsertSoftwareInstallRequest(ctx, host.ID, installer.InstallerID, fleet.HostSoftwareInstallOptions{ + SelfService: true, + }) return ctxerr.Wrap(ctx, err, "inserting self-service software install request") } @@ -1852,7 +1842,9 @@ func (svc *Service) SelfServiceInstallSoftwareTitle(ctx context.Context, host *f platform := host.FleetPlatform() mobileAppleDevice := fleet.AppleDevicePlatform(platform) == fleet.IOSPlatform || fleet.AppleDevicePlatform(platform) == fleet.IPadOSPlatform - _, err = svc.installSoftwareFromVPP(ctx, host, vppApp, mobileAppleDevice || fleet.AppleDevicePlatform(platform) == fleet.MacOSPlatform, true) + _, err = svc.installSoftwareFromVPP(ctx, host, vppApp, mobileAppleDevice || fleet.AppleDevicePlatform(platform) == fleet.MacOSPlatform, fleet.HostSoftwareInstallOptions{ + SelfService: true, + }) return err } diff --git a/ee/server/service/software_installers_test.go b/ee/server/service/software_installers_test.go index 769d91b7aed9..db97d917c8ab 100644 --- a/ee/server/service/software_installers_test.go +++ b/ee/server/service/software_installers_test.go @@ -105,7 +105,7 @@ func TestInstallUninstallAuth(t *testing.T) { ds.GetHostLastInstallDataFunc = func(ctx context.Context, hostID uint, installerID uint) (*fleet.HostLastInstallData, error) { return nil, nil } - ds.InsertSoftwareInstallRequestFunc = func(ctx context.Context, hostID uint, softwareInstallerID uint, selfService bool, policyID *uint) (string, + ds.InsertSoftwareInstallRequestFunc = func(ctx context.Context, hostID uint, softwareInstallerID uint, opts fleet.HostSoftwareInstallOptions) (string, error, ) { return "request_id", nil @@ -113,12 +113,6 @@ func TestInstallUninstallAuth(t *testing.T) { ds.GetAnyScriptContentsFunc = func(ctx context.Context, id uint) ([]byte, error) { return []byte("script"), nil } - ds.NewInternalScriptExecutionRequestFunc = func(ctx context.Context, request *fleet.HostScriptRequestPayload) (*fleet.HostScriptResult, - error) { - return &fleet.HostScriptResult{ - ExecutionID: "execution_id", - }, nil - } ds.InsertSoftwareUninstallRequestFunc = func(ctx context.Context, executionID string, hostID uint, softwareInstallerID uint) error { return nil } diff --git a/frontend/__mocks__/activityMock.ts b/frontend/__mocks__/activityMock.ts index cdf84c1b8b79..04458588bda3 100644 --- a/frontend/__mocks__/activityMock.ts +++ b/frontend/__mocks__/activityMock.ts @@ -7,6 +7,7 @@ const DEFAULT_ACTIVITY_MOCK: IActivity = { actor_id: 1, actor_gravatar: "", actor_email: "test@example.com", + fleet_initiated: false, type: ActivityType.EditedAgentOptions, }; diff --git a/frontend/components/ActivityItem/ActivityItem.tsx b/frontend/components/ActivityItem/ActivityItem.tsx index 01451e703ea8..6ab08be8aba9 100644 --- a/frontend/components/ActivityItem/ActivityItem.tsx +++ b/frontend/components/ActivityItem/ActivityItem.tsx @@ -2,7 +2,7 @@ import React from "react"; import ReactTooltip from "react-tooltip"; import classnames from "classnames"; -import { ActivityType, IActivity, IActivityDetails } from "interfaces/activity"; +import { IActivity, IActivityDetails } from "interfaces/activity"; import { addGravatarUrlToResource, internationalTimeFormat, @@ -108,20 +108,6 @@ const ActivityItem = ({ onCancel(); }; - // TODO: remove this once we have a proper way of handling "Fleet-initiated" activities in - // the backend. For now, if all these fields are empty, then we assume it was - // Fleet-initiated. - let fleetInitiated = false; - if ( - !activity.actor_email && - !activity.actor_full_name && - (activity.type === ActivityType.InstalledSoftware || - activity.type === ActivityType.InstalledAppStoreApp || - activity.type === ActivityType.RanScript) - ) { - fleetInitiated = true; - } - return (
@@ -131,7 +117,7 @@ const ActivityItem = ({ user={{ gravatar_url }} size="small" hasWhiteBackground - useFleetAvatar={fleetInitiated} + useFleetAvatar={activity.fleet_initiated} />
diff --git a/frontend/interfaces/activity.ts b/frontend/interfaces/activity.ts index e31355bc4d6d..e2f9d4797301 100644 --- a/frontend/interfaces/activity.ts +++ b/frontend/interfaces/activity.ts @@ -127,6 +127,7 @@ export interface IActivity { actor_gravatar: string; actor_email?: string; type: ActivityType; + fleet_initiated: boolean; details?: IActivityDetails; } diff --git a/frontend/pages/DashboardPage/cards/ActivityFeed/GlobalActivityItem/GlobalActivityItem.tsx b/frontend/pages/DashboardPage/cards/ActivityFeed/GlobalActivityItem/GlobalActivityItem.tsx index 6d6004b1c464..5c766ca41c35 100644 --- a/frontend/pages/DashboardPage/cards/ActivityFeed/GlobalActivityItem/GlobalActivityItem.tsx +++ b/frontend/pages/DashboardPage/cards/ActivityFeed/GlobalActivityItem/GlobalActivityItem.tsx @@ -1278,18 +1278,6 @@ const GlobalActivityItem = ({ }: IActivityItemProps) => { const hasDetails = ACTIVITIES_WITH_DETAILS.has(activity.type); - // Add the "Fleet" name to the activity if needed. - // TODO: remove/refactor this once we have "fleet-initiated" activities. - if ( - !activity.actor_email && - !activity.actor_full_name && - (activity.type === ActivityType.InstalledSoftware || - activity.type === ActivityType.InstalledAppStoreApp || - activity.type === ActivityType.RanScript) - ) { - activity.actor_full_name = "Fleet"; - } - const renderActivityPrefix = () => { const DEFAULT_ACTOR_DISPLAY = {activity.actor_full_name} ; diff --git a/frontend/pages/hosts/details/cards/Activity/PastActivityFeed/PastActivityFeed.tsx b/frontend/pages/hosts/details/cards/Activity/PastActivityFeed/PastActivityFeed.tsx index cb88c0d35a1a..4298b538f8f2 100644 --- a/frontend/pages/hosts/details/cards/Activity/PastActivityFeed/PastActivityFeed.tsx +++ b/frontend/pages/hosts/details/cards/Activity/PastActivityFeed/PastActivityFeed.tsx @@ -1,6 +1,6 @@ import React from "react"; -import { ActivityType, IHostPastActivity } from "interfaces/activity"; +import { IHostPastActivity } from "interfaces/activity"; import { IHostPastActivitiesResponse } from "services/entities/activities"; // @ts-ignore @@ -54,18 +54,6 @@ const PastActivityFeed = ({
{activitiesList.map((activity: IHostPastActivity) => { - // TODO: remove this once we have a proper way of handling "Fleet-initiated" activities in - // the backend. For now, if all these fields are empty, then we assume it was - // Fleet-initiated. - if ( - !activity.actor_email && - !activity.actor_full_name && - (activity.type === ActivityType.InstalledSoftware || - activity.type === ActivityType.InstalledAppStoreApp || - activity.type === ActivityType.RanScript) - ) { - activity.actor_full_name = "Fleet"; - } const ActivityItemComponent = pastActivityComponentMap[activity.type]; return (
{activitiesList.map((activity: IHostUpcomingActivity) => { - // TODO: remove this once we have a proper way of handling "Fleet-initiated" activities in - // the backend. For now, if all these fields are empty, then we assume it was - // Fleet-initiated. - if ( - !activity.actor_email && - !activity.actor_full_name && - (activity.type === ActivityType.InstalledSoftware || - activity.type === ActivityType.InstalledAppStoreApp || - activity.type === ActivityType.RanScript) - ) { - activity.actor_full_name = "Fleet"; - } const ActivityItemComponent = upcomingActivityComponentMap[activity.type]; return ( diff --git a/server/datastore/mysql/activities.go b/server/datastore/mysql/activities.go index 3786566d0250..99742750fdb2 100644 --- a/server/datastore/mysql/activities.go +++ b/server/datastore/mysql/activities.go @@ -8,9 +8,9 @@ import ( "strings" "time" - "github.com/fleetdm/fleet/v4/pkg/scripts" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/mdm" "github.com/go-kit/log/level" "github.com/jmoiron/sqlx" ) @@ -58,6 +58,29 @@ func (ds *Datastore) NewActivity( cols = append(cols, "user_email") } + if vppAct, ok := activity.(fleet.ActivityInstalledAppStoreApp); ok { + // NOTE: ideally this would be called in the same transaction as storing + // the nanomdm command results, but the current design doesn't allow for + // that with the nano store being a distinct entity to our datastore (we + // should get rid of that distinction eventually, we've broken it already + // in some places and it doesn't bring much benefit anymore). + // + // Instead, this gets called from CommandAndReportResults, which is + // executed after the results have been saved in nano, but we already + // accept this non-transactional fact for many other states we manage in + // Fleet (wipe, lock results, setup experience results, etc. - see all + // critical data that gets updated in CommandAndReportResults) so there's + // no reason to treat the unified queue differently. + // + // This place here is a bit hacky but perfect for VPP apps as the activity + // gets created only when the MDM command status is in a final state + // (success or failure), which is exactly when we want to activate the next + // activity. + if _, err := ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), vppAct.HostID, vppAct.CommandUUID); err != nil { + return ctxerr.Wrap(ctx, err, "activate next activity from VPP app install") + } + } + return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { const insertActStmt = `INSERT INTO activities (%s) VALUES (%s)` sql := fmt.Sprintf(insertActStmt, strings.Join(cols, ","), strings.Repeat("?,", len(cols)-1)+"?") @@ -239,220 +262,186 @@ func (ds *Datastore) MarkActivitiesAsStreamed(ctx context.Context, activityIDs [ // number of distinct tables that are task-specific (such as scripts to run, // software to install, etc.) and provides a unified view of those upcoming // tasks. -func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint, opt fleet.ListOptions) ([]*fleet.Activity, *fleet.PaginationMetadata, error) { +func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint, opt fleet.ListOptions) ([]*fleet.UpcomingActivity, *fleet.PaginationMetadata, error) { // NOTE: Be sure to update both the count (here) and list statements (below) // if the query condition is modified. - countStmts := []string{ - `SELECT - COUNT(*) c - FROM host_script_results hsr - LEFT OUTER JOIN - host_software_installs hsi ON hsi.execution_id = hsr.execution_id - WHERE hsr.host_id = :host_id AND - hsr.host_deleted_at IS NULL AND - exit_code IS NULL AND - hsi.execution_id IS NULL AND - (sync_request = 0 OR hsr.created_at >= DATE_SUB(NOW(), INTERVAL :max_wait_time SECOND))`, - `SELECT - COUNT(*) c - FROM host_software_installs hsi - WHERE hsi.host_id = :host_id AND hsi.software_installer_id IS NOT NULL AND - hsi.host_deleted_at IS NULL AND - hsi.status = :software_status_install_pending`, - `SELECT - COUNT(*) c - FROM host_software_installs hsi - WHERE hsi.host_id = :host_id AND hsi.software_installer_id IS NOT NULL AND - hsi.host_deleted_at IS NULL AND - hsi.status = :software_status_uninstall_pending`, - ` - SELECT - COUNT(*) c - FROM nano_view_queue nvq - JOIN host_vpp_software_installs hvsi ON nvq.command_uuid = hvsi.command_uuid - WHERE hvsi.host_id = :host_id AND nvq.status IS NULL - `, - } + + const countStmt = `SELECT + COUNT(*) c + FROM upcoming_activities + WHERE host_id = ?` var count uint - countStmt := `SELECT SUM(c) FROM ( ` + strings.Join(countStmts, " UNION ALL ") + ` ) AS counts` - - seconds := int(scripts.MaxServerWaitTime.Seconds()) - countStmt, args, err := sqlx.Named(countStmt, map[string]any{ - "host_id": hostID, - "max_wait_time": seconds, - "software_status_install_pending": fleet.SoftwareInstallPending, - "software_status_uninstall_pending": fleet.SoftwareUninstallPending, - }) - if err != nil { - return nil, nil, ctxerr.Wrap(ctx, err, "build count query from named args") - } - if err := sqlx.GetContext(ctx, ds.reader(ctx), &count, countStmt, args...); err != nil { + if err := sqlx.GetContext(ctx, ds.reader(ctx), &count, countStmt, hostID); err != nil { return nil, nil, ctxerr.Wrap(ctx, err, "count upcoming activities") } if count == 0 { - return []*fleet.Activity{}, &fleet.PaginationMetadata{}, nil + return []*fleet.UpcomingActivity{}, &fleet.PaginationMetadata{}, nil } // NOTE: Be sure to update both the count (above) and list statements (below) // if the query condition is modified. + listStmts := []string{ // list pending scripts `SELECT - hsr.execution_id as uuid, - IF(hsr.policy_id IS NOT NULL, 'Fleet', u.name) as name, + ua.execution_id as uuid, + IF(ua.fleet_initiated, 'Fleet', COALESCE(u.name, ua.payload->>'$.user.name')) as name, u.id as user_id, - u.gravatar_url as gravatar_url, - u.email as user_email, + COALESCE(u.gravatar_url, ua.payload->>'$.user.gravatar_url') as gravatar_url, + COALESCE(u.email, ua.payload->>'$.user.email') as user_email, :ran_script_type as activity_type, - hsr.created_at as created_at, + ua.created_at as created_at, JSON_OBJECT( - 'host_id', hsr.host_id, + 'host_id', ua.host_id, 'host_display_name', COALESCE(hdn.display_name, ''), - 'script_name', COALESCE(ses.name, COALESCE(scr.name, '')), - 'script_execution_id', hsr.execution_id, - 'async', NOT hsr.sync_request, - 'policy_id', hsr.policy_id, - 'policy_name', p.name - ) as details + 'script_name', COALESCE(ses.name, scr.name, ''), + 'script_execution_id', ua.execution_id, + 'async', NOT ua.payload->'$.sync_request', + 'policy_id', sua.policy_id, + 'policy_name', p.name + ) as details, + IF(ua.activated_at IS NULL, 0, 1) as topmost, + ua.priority as priority, + ua.fleet_initiated as fleet_initiated, + IF(ua.activated_at IS NULL, 1, 0) as cancellable FROM - host_script_results hsr - LEFT OUTER JOIN - users u ON u.id = hsr.user_id + upcoming_activities ua + INNER JOIN + script_upcoming_activities sua ON sua.upcoming_activity_id = ua.id LEFT OUTER JOIN - policies p ON p.id = hsr.policy_id + users u ON u.id = ua.user_id LEFT OUTER JOIN - host_display_names hdn ON hdn.host_id = hsr.host_id + policies p ON p.id = sua.policy_id LEFT OUTER JOIN - scripts scr ON scr.id = hsr.script_id + host_display_names hdn ON hdn.host_id = ua.host_id LEFT OUTER JOIN - host_software_installs hsi ON hsi.execution_id = hsr.execution_id + scripts scr ON scr.id = sua.script_id LEFT OUTER JOIN - setup_experience_scripts ses ON ses.id = hsr.setup_experience_script_id + setup_experience_scripts ses ON ses.id = sua.setup_experience_script_id WHERE - hsr.host_id = :host_id AND - hsr.host_deleted_at IS NULL AND - hsr.exit_code IS NULL AND - ( - hsr.sync_request = 0 OR - hsr.created_at >= DATE_SUB(NOW(), INTERVAL :max_wait_time SECOND) - ) AND - hsi.execution_id IS NULL + ua.host_id = :host_id AND + ua.activity_type = 'script' `, // list pending software installs `SELECT - hsi.execution_id as uuid, - -- policies with automatic installers generate a host_software_installs with (user_id=NULL,self_service=0), - -- so we mark those as "Fleet" - IF(hsi.user_id IS NULL AND NOT hsi.self_service, 'Fleet', u.name) AS name, - hsi.user_id as user_id, - u.gravatar_url as gravatar_url, - u.email AS user_email, + ua.execution_id as uuid, + IF(ua.fleet_initiated, 'Fleet', COALESCE(u.name, ua.payload->>'$.user.name')) AS name, + ua.user_id as user_id, + COALESCE(u.gravatar_url, ua.payload->>'$.user.gravatar_url') as gravatar_url, + COALESCE(u.email, ua.payload->>'$.user.email') as user_email, :installed_software_type as activity_type, - hsi.created_at as created_at, + ua.created_at as created_at, JSON_OBJECT( - 'host_id', hsi.host_id, + 'host_id', ua.host_id, 'host_display_name', COALESCE(hdn.display_name, ''), - 'software_title', COALESCE(st.name, ''), - 'software_package', si.filename, - 'install_uuid', hsi.execution_id, - 'status', CAST(hsi.status AS CHAR), - 'self_service', hsi.self_service IS TRUE, - 'policy_id', hsi.policy_id, - 'policy_name', p.name - ) as details + 'software_title', COALESCE(st.name, ua.payload->>'$.software_title_name', ''), + 'software_package', COALESCE(si.filename, ua.payload->>'$.installer_filename', ''), + 'install_uuid', ua.execution_id, + 'status', 'pending_install', + 'self_service', ua.payload->'$.self_service' IS TRUE, + 'policy_id', siua.policy_id, + 'policy_name', p.name + ) as details, + IF(ua.activated_at IS NULL, 0, 1) as topmost, + ua.priority as priority, + ua.fleet_initiated as fleet_initiated, + IF(ua.activated_at IS NULL, 1, 0) as cancellable FROM - host_software_installs hsi + upcoming_activities ua INNER JOIN - software_installers si ON si.id = hsi.software_installer_id + software_install_upcoming_activities siua ON siua.upcoming_activity_id = ua.id + LEFT OUTER JOIN + software_installers si ON si.id = siua.software_installer_id LEFT OUTER JOIN software_titles st ON st.id = si.title_id LEFT OUTER JOIN - users u ON u.id = hsi.user_id + users u ON u.id = ua.user_id LEFT OUTER JOIN - policies p ON p.id = hsi.policy_id + policies p ON p.id = siua.policy_id LEFT OUTER JOIN - host_display_names hdn ON hdn.host_id = hsi.host_id + host_display_names hdn ON hdn.host_id = ua.host_id WHERE - hsi.host_id = :host_id AND - hsi.host_deleted_at IS NULL AND - hsi.status = :software_status_install_pending + ua.host_id = :host_id AND + ua.activity_type = 'software_install' `, // list pending software uninstalls `SELECT - hsi.execution_id as uuid, - -- policies with automatic installers generate a host_software_installs with (user_id=NULL,self_service=0), - -- so we mark those as "Fleet" - IF(hsi.user_id IS NULL AND NOT hsi.self_service, 'Fleet', u.name) AS name, - hsi.user_id as user_id, - u.gravatar_url as gravatar_url, - u.email AS user_email, + ua.execution_id as uuid, + IF(ua.fleet_initiated, 'Fleet', COALESCE(u.name, ua.payload->>'$.user.name')) AS name, + ua.user_id as user_id, + COALESCE(u.gravatar_url, ua.payload->>'$.user.gravatar_url') as gravatar_url, + COALESCE(u.email, ua.payload->>'$.user.email') as user_email, :uninstalled_software_type as activity_type, - hsi.created_at as created_at, + ua.created_at as created_at, JSON_OBJECT( - 'host_id', hsi.host_id, + 'host_id', ua.host_id, 'host_display_name', COALESCE(hdn.display_name, ''), - 'software_title', COALESCE(st.name, ''), - 'script_execution_id', hsi.execution_id, - 'status', CAST(hsi.status AS CHAR), - 'policy_id', hsi.policy_id, - 'policy_name', p.name - ) as details + 'software_title', COALESCE(st.name, ua.payload->>'$.software_title_name', ''), + 'script_execution_id', ua.execution_id, + 'status', 'pending_uninstall', + 'policy_id', siua.policy_id, + 'policy_name', p.name + ) as details, + IF(ua.activated_at IS NULL, 0, 1) as topmost, + ua.priority as priority, + ua.fleet_initiated as fleet_initiated, + IF(ua.activated_at IS NULL, 1, 0) as cancellable FROM - host_software_installs hsi + upcoming_activities ua INNER JOIN - software_installers si ON si.id = hsi.software_installer_id + software_install_upcoming_activities siua ON siua.upcoming_activity_id = ua.id + LEFT OUTER JOIN + software_installers si ON si.id = siua.software_installer_id LEFT OUTER JOIN software_titles st ON st.id = si.title_id LEFT OUTER JOIN - users u ON u.id = hsi.user_id + users u ON u.id = ua.user_id LEFT OUTER JOIN - policies p ON p.id = hsi.policy_id + policies p ON p.id = siua.policy_id LEFT OUTER JOIN - host_display_names hdn ON hdn.host_id = hsi.host_id + host_display_names hdn ON hdn.host_id = ua.host_id WHERE - hsi.host_id = :host_id AND - hsi.host_deleted_at IS NULL AND - hsi.status = :software_status_uninstall_pending + ua.host_id = :host_id AND + activity_type = 'software_uninstall' + `, + `SELECT + ua.execution_id AS uuid, + IF(ua.fleet_initiated, 'Fleet', COALESCE(u.name, ua.payload->>'$.user.name')) AS name, + u.id AS user_id, + COALESCE(u.gravatar_url, ua.payload->>'$.user.gravatar_url') as gravatar_url, + COALESCE(u.email, ua.payload->>'$.user.email') as user_email, + :installed_app_store_app_type AS activity_type, + ua.created_at AS created_at, + JSON_OBJECT( + 'host_id', ua.host_id, + 'host_display_name', hdn.display_name, + 'software_title', st.name, + 'app_store_id', vaua.adam_id, + 'command_uuid', ua.execution_id, + 'self_service', ua.payload->'$.self_service' IS TRUE, + 'status', 'pending_install' + ) AS details, + IF(ua.activated_at IS NULL, 0, 1) as topmost, + ua.priority as priority, + ua.fleet_initiated as fleet_initiated, + IF(ua.activated_at IS NULL, 1, 0) as cancellable + FROM + upcoming_activities ua + INNER JOIN + vpp_app_upcoming_activities vaua ON vaua.upcoming_activity_id = ua.id + LEFT OUTER JOIN + users u ON ua.user_id = u.id + LEFT OUTER JOIN + host_display_names hdn ON hdn.host_id = ua.host_id + LEFT OUTER JOIN + vpp_apps vpa ON vaua.adam_id = vpa.adam_id AND vaua.platform = vpa.platform + LEFT OUTER JOIN + software_titles st ON st.id = vpa.title_id + WHERE + ua.host_id = :host_id AND + ua.activity_type = 'vpp_app_install' `, - // list pending VPP installs - ` -SELECT - hvsi.command_uuid AS uuid, - -- policies with automatic installers generate a host_vpp_software_installs with (user_id=NULL,self_service=0), - -- so we mark those as "Fleet" - IF(hvsi.user_id IS NULL AND NOT hvsi.self_service, 'Fleet', u.name) AS name, - u.id AS user_id, - u.gravatar_url as gravatar_url, - u.email as user_email, - :installed_app_store_app_type AS activity_type, - hvsi.created_at AS created_at, - JSON_OBJECT( - 'host_id', hvsi.host_id, - 'host_display_name', hdn.display_name, - 'software_title', st.name, - 'app_store_id', hvsi.adam_id, - 'command_uuid', hvsi.command_uuid, - 'self_service', hvsi.self_service IS TRUE, - -- status is always pending because only pending MDM commands are upcoming. - 'status', :software_status_install_pending - ) AS details -FROM - host_vpp_software_installs hvsi -INNER JOIN - nano_view_queue nvq ON nvq.command_uuid = hvsi.command_uuid -LEFT OUTER JOIN - users u ON hvsi.user_id = u.id -LEFT OUTER JOIN - host_display_names hdn ON hdn.host_id = hvsi.host_id -LEFT OUTER JOIN - vpp_apps vpa ON hvsi.adam_id = vpa.adam_id AND hvsi.platform = vpa.platform -LEFT OUTER JOIN - software_titles st ON st.id = vpa.title_id -WHERE - nvq.status IS NULL - AND hvsi.host_id = :host_id -`, } listStmt := ` @@ -464,28 +453,38 @@ WHERE user_email, activity_type, created_at, - details - FROM ( ` + strings.Join(listStmts, " UNION ALL ") + ` ) AS upcoming ` - listStmt, args, err = sqlx.Named(listStmt, map[string]any{ - "host_id": hostID, - "ran_script_type": fleet.ActivityTypeRanScript{}.ActivityName(), - "installed_software_type": fleet.ActivityTypeInstalledSoftware{}.ActivityName(), - "uninstalled_software_type": fleet.ActivityTypeUninstalledSoftware{}.ActivityName(), - "installed_app_store_app_type": fleet.ActivityInstalledAppStoreApp{}.ActivityName(), - "max_wait_time": seconds, - "software_status_install_pending": fleet.SoftwareInstallPending, - "software_status_uninstall_pending": fleet.SoftwareUninstallPending, + details, + fleet_initiated, + cancellable + FROM ( ` + strings.Join(listStmts, " UNION ALL ") + ` ) AS upcoming + ORDER BY topmost DESC, priority DESC, created_at ASC` + + listStmt, args, err := sqlx.Named(listStmt, map[string]any{ + "host_id": hostID, + "ran_script_type": fleet.ActivityTypeRanScript{}.ActivityName(), + "installed_software_type": fleet.ActivityTypeInstalledSoftware{}.ActivityName(), + "uninstalled_software_type": fleet.ActivityTypeUninstalledSoftware{}.ActivityName(), + "installed_app_store_app_type": fleet.ActivityInstalledAppStoreApp{}.ActivityName(), }) if err != nil { return nil, nil, ctxerr.Wrap(ctx, err, "build list query from named args") } + + // the ListOptions supported for this query are limited, only the pagination + // OFFSET and LIMIT can be added, so it's fine to have the ORDER BY already + // in the query before calling this (enforced at the server layer). stmt, args := appendListOptionsWithCursorToSQL(listStmt, args, &opt) - var activities []*fleet.Activity + var activities []*fleet.UpcomingActivity if err := sqlx.SelectContext(ctx, ds.reader(ctx), &activities, stmt, args...); err != nil { return nil, nil, ctxerr.Wrap(ctx, err, "select upcoming activities") } + // first activity (next one to execute) is always non-cancellable, per spec + if len(activities) > 0 && opt.Page == 0 { + activities[0].Cancellable = false + } + metaData := &fleet.PaginationMetadata{HasPreviousResults: opt.Page > 0, TotalResults: count} if len(activities) > int(opt.PerPage) { //nolint:gosec // dismiss G115 metaData.HasNextResults = true @@ -623,3 +622,460 @@ func (ds *Datastore) CleanupActivitiesAndAssociatedData(ctx context.Context, max return nil } + +// This function activates the next upcoming activity, if any, for the specified host. +// It does a few things to achieve this: +// - If there was an activity already marked as activated (activated_at is +// not NULL) and fromCompletedExecID is provided, it deletes it, as calling +// this function means that this activated activity is now completed (in a +// final state, either success or failure). +// - If no other activity is still activated and there is an upcoming +// activity to activate next, it does so, respecting the priority and enqueue +// order. Activation consists of inserting the activity in its respective +// table, e.g. `host_script_results` for scripts, `host_sofware_installs` for +// software installs, `host_vpp_software_installs` and nano command queue for +// VPP installs; and setting the activated_at timestamp in the +// `upcoming_activities` table. +// - As an optimization for MDM, if the activity type is `vpp_app_install` +// and the next few upcoming activities are all of this type, they are +// batch-activated together (up to a limit) to reduce the processing +// latency and number of push notifications to send to this host. +// +// When called after receiving results for an activity, the fromCompletedExecID +// argument identifies that completed activity. +func (ds *Datastore) activateNextUpcomingActivity(ctx context.Context, tx sqlx.ExtContext, hostID uint, fromCompletedExecID string) (activatedExecIDs []string, err error) { + const maxMDMCommandActivations = 5 + + const deleteCompletedStmt = ` +DELETE FROM upcoming_activities +WHERE + host_id = ? AND + activated_at IS NOT NULL AND + execution_id = ? +` + + const findNextStmt = ` +SELECT + execution_id, + activity_type, + activated_at, + IF(activated_at IS NULL, 0, 1) as topmost, + priority +FROM + upcoming_activities +WHERE + host_id = ? + %s +ORDER BY topmost DESC, priority DESC, created_at ASC +LIMIT ? +` + + const findNextSpecificExecIDsClause = ` AND execution_id IN (?) ` + + const markActivatedStmt = ` +UPDATE upcoming_activities +SET + activated_at = NOW() +WHERE + host_id = ? AND + execution_id IN (?) +` + + // first we delete the completed activity, if any + if fromCompletedExecID != "" { + if _, err := tx.ExecContext(ctx, deleteCompletedStmt, hostID, fromCompletedExecID); err != nil { + return nil, ctxerr.Wrap(ctx, err, "delete completed upcoming activity") + } + } + + // next we look for an upcoming activity to activate + type nextActivity struct { + ExecutionID string `db:"execution_id"` + ActivityType string `db:"activity_type"` + ActivatedAt *time.Time `db:"activated_at"` + Topmost bool `db:"topmost"` + Priority int `db:"priority"` + } + var nextActivities []nextActivity + stmt, args := fmt.Sprintf(findNextStmt, ""), []any{hostID, maxMDMCommandActivations} + if len(ds.testActivateSpecificNextActivities) > 0 { + stmt, args, err = sqlx.In(fmt.Sprintf(findNextStmt, findNextSpecificExecIDsClause), + hostID, ds.testActivateSpecificNextActivities, len(ds.testActivateSpecificNextActivities)) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "prepare find next upcoming activities statement with test execution ids") + } + } + if err := sqlx.SelectContext(ctx, tx, &nextActivities, stmt, args...); err != nil { + return nil, ctxerr.Wrap(ctx, err, "find next upcoming activities to activate") + } + + var toActivate []nextActivity + for _, act := range nextActivities { + if act.ActivatedAt != nil { + // there are still activated activities, do not activate more + break + } + if len(toActivate) > 0 { + // we already identified one to activate, allow more only if they are a) + // the same type, b) that type is vpp_app_install, c) the same priority. + // The reason for that is to batch-activate MDM commands to reduce + // latency and push notifications required, and the same priority check + // is because we can't enforce the ordering of commands if they don't + // share the same priority (we transfer the created_at timestamp to the + // nano queue, which guarantees same order of processing for activities + // with the same priority). + if toActivate[0].ActivityType != act.ActivityType || + toActivate[0].ActivityType != "vpp_app_install" || + toActivate[0].Priority != act.Priority { + break + } + } + toActivate = append(toActivate, act) + activatedExecIDs = append(activatedExecIDs, act.ExecutionID) + } + + if len(toActivate) == 0 { + return nil, nil + } + + // activate the next activities as required for its activity type + var fn func(context.Context, sqlx.ExtContext, uint, []string) error + switch actType := toActivate[0].ActivityType; actType { + case "script": + fn = ds.activateNextScriptActivity + case "software_install": + fn = ds.activateNextSoftwareInstallActivity + case "software_uninstall": + fn = ds.activateNextSoftwareUninstallActivity + case "vpp_app_install": + fn = ds.activateNextVPPAppInstallActivity + default: + return nil, ctxerr.Errorf(ctx, "unsupported activity type %s", actType) + } + if err := fn(ctx, tx, hostID, activatedExecIDs); err != nil { + return nil, ctxerr.Wrap(ctx, err, "activate next activities") + } + + // finally, mark the activities as activated + stmt, args, err = sqlx.In(markActivatedStmt, hostID, activatedExecIDs) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "prepare statement to mark upcoming activities as activated") + } + if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { + return nil, ctxerr.Wrap(ctx, err, "mark upcoming activities as activated") + } + return activatedExecIDs, nil +} + +func (ds *Datastore) activateNextScriptActivity(ctx context.Context, tx sqlx.ExtContext, hostID uint, execIDs []string) error { + const insStmt = ` +INSERT INTO + host_script_results +(host_id, execution_id, script_content_id, output, script_id, policy_id, + user_id, sync_request, setup_experience_script_id, is_internal) +SELECT + ua.host_id, + ua.execution_id, + sua.script_content_id, + '', + sua.script_id, + sua.policy_id, + ua.user_id, + COALESCE(ua.payload->'$.sync_request', 0), + sua.setup_experience_script_id, + COALESCE(ua.payload->'$.is_internal', 0) +FROM + upcoming_activities ua + INNER JOIN script_upcoming_activities sua + ON sua.upcoming_activity_id = ua.id +WHERE + ua.host_id = ? AND + ua.execution_id IN (?) +ORDER BY + ua.priority DESC, ua.created_at ASC +` + + // sanity-check that there's something to activate + if len(execIDs) == 0 { + return nil + } + stmt, args, err := sqlx.In(insStmt, hostID, execIDs) + if err != nil { + return ctxerr.Wrap(ctx, err, "prepare insert to activate scripts") + } + if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "insert to activate scripts") + } + return nil +} + +func (ds *Datastore) activateNextSoftwareInstallActivity(ctx context.Context, tx sqlx.ExtContext, hostID uint, execIDs []string) error { + const insStmt = ` +INSERT INTO host_software_installs + (execution_id, host_id, software_installer_id, user_id, self_service, + policy_id, installer_filename, version, software_title_id, software_title_name) +SELECT + ua.execution_id, + ua.host_id, + siua.software_installer_id, + ua.user_id, + COALESCE(ua.payload->'$.self_service', 0), + siua.policy_id, + COALESCE(ua.payload->>'$.installer_filename', '[deleted installer]'), + COALESCE(ua.payload->>'$.version', 'unknown'), + siua.software_title_id, + COALESCE(ua.payload->>'$.software_title_name', '[deleted title]') +FROM + upcoming_activities ua + INNER JOIN software_install_upcoming_activities siua + ON siua.upcoming_activity_id = ua.id +WHERE + ua.host_id = ? AND + ua.execution_id IN (?) +ORDER BY + ua.priority DESC, ua.created_at ASC +` + + // sanity-check that there's something to activate + if len(execIDs) == 0 { + return nil + } + stmt, args, err := sqlx.In(insStmt, hostID, execIDs) + if err != nil { + return ctxerr.Wrap(ctx, err, "prepare insert to activate software installs") + } + if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "insert to activate software installs") + } + return nil +} + +func (ds *Datastore) activateNextSoftwareUninstallActivity(ctx context.Context, tx sqlx.ExtContext, hostID uint, execIDs []string) error { + const insScriptStmt = ` +INSERT INTO + host_script_results +(host_id, execution_id, script_content_id, output, user_id, is_internal) +SELECT + ua.host_id, + ua.execution_id, + si.uninstall_script_content_id, + '', + ua.user_id, + 1 +FROM + upcoming_activities ua + INNER JOIN software_install_upcoming_activities siua + ON siua.upcoming_activity_id = ua.id + INNER JOIN software_installers si + ON si.id = siua.software_installer_id +WHERE + ua.host_id = ? AND + ua.execution_id IN (?) +ORDER BY + ua.priority DESC, ua.created_at ASC +` + + const insSwStmt = ` +INSERT INTO + host_software_installs +(execution_id, host_id, software_installer_id, user_id, uninstall, installer_filename, + software_title_id, software_title_name, version) +SELECT + ua.execution_id, + ua.host_id, + siua.software_installer_id, + ua.user_id, + 1, -- uninstall + '', -- no installer_filename for uninstalls + siua.software_title_id, + COALESCE(ua.payload->>'$.software_title_name', '[deleted title]'), + 'unknown' +FROM + upcoming_activities ua + INNER JOIN software_install_upcoming_activities siua + ON siua.upcoming_activity_id = ua.id +WHERE + ua.host_id = ? AND + ua.execution_id IN (?) +ORDER BY + ua.priority DESC, ua.created_at ASC +` + // sanity-check that there's something to activate + if len(execIDs) == 0 { + return nil + } + + stmt, args, err := sqlx.In(insScriptStmt, hostID, execIDs) + if err != nil { + return ctxerr.Wrap(ctx, err, "prepare insert script to activate software uninstalls") + } + if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "insert script to activate software uninstalls") + } + + stmt, args, err = sqlx.In(insSwStmt, hostID, execIDs) + if err != nil { + return ctxerr.Wrap(ctx, err, "prepare insert software to activate software uninstalls") + } + if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "insert software to activate software uninstalls") + } + return nil +} + +func (ds *Datastore) activateNextVPPAppInstallActivity(ctx context.Context, tx sqlx.ExtContext, hostID uint, execIDs []string) error { + const insStmt = ` +INSERT INTO + host_vpp_software_installs +(host_id, adam_id, platform, command_uuid, + user_id, associated_event_id, self_service, policy_id) +SELECT + ua.host_id, + vaua.adam_id, + vaua.platform, + ua.execution_id, + ua.user_id, + ua.payload->>'$.associated_event_id', + COALESCE(ua.payload->'$.self_service', 0), + vaua.policy_id +FROM + upcoming_activities ua + INNER JOIN vpp_app_upcoming_activities vaua + ON vaua.upcoming_activity_id = ua.id +WHERE + ua.host_id = ? AND + ua.execution_id IN (?) +ORDER BY + ua.priority DESC, ua.created_at ASC +` + + const getHostUUIDStmt = ` +SELECT + uuid +FROM + hosts +WHERE + id = ? +` + + const insCmdStmt = ` +INSERT INTO + nano_commands +(command_uuid, request_type, command, subtype) +SELECT + ua.execution_id, + 'InstallApplication', + CONCAT(:raw_cmd_part1, vaua.adam_id, :raw_cmd_part2, ua.execution_id, :raw_cmd_part3), + :subtype +FROM + upcoming_activities ua + INNER JOIN vpp_app_upcoming_activities vaua + ON vaua.upcoming_activity_id = ua.id +WHERE + ua.host_id = :host_id AND + ua.execution_id IN (:execution_ids) +` + + const rawCmdPart1 = ` + + + + Command + + ManagementFlags + 0 + Options + + PurchaseMethod + 1 + + RequestType + InstallApplication + iTunesStoreID + ` + + const rawCmdPart2 = ` + + CommandUUID + ` + + const rawCmdPart3 = ` + +` + + const insNanoQueueStmt = ` +INSERT INTO + nano_enrollment_queue +(id, command_uuid, created_at) +SELECT + ?, + execution_id, + created_at -- force same timestamp to keep ordering +FROM + upcoming_activities +WHERE + host_id = ? AND + execution_id IN (?) +ORDER BY + priority DESC, created_at ASC +` + + // sanity-check that there's something to activate + if len(execIDs) == 0 { + return nil + } + + // get the host uuid, requires for the nano tables + var hostUUID string + if err := sqlx.GetContext(ctx, tx, &hostUUID, getHostUUIDStmt, hostID); err != nil { + return ctxerr.Wrap(ctx, err, "get host uuid") + } + + // insert the host vpp app row + stmt, args, err := sqlx.In(insStmt, hostID, execIDs) + if err != nil { + return ctxerr.Wrap(ctx, err, "prepare insert to activate vpp apps") + } + if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "insert to activate vpp apps") + } + + // insert the nano command + namedArgs := map[string]any{ + "raw_cmd_part1": rawCmdPart1, + "raw_cmd_part2": rawCmdPart2, + "raw_cmd_part3": rawCmdPart3, + "subtype": mdm.CommandSubtypeNone, + "host_id": hostID, + "execution_ids": execIDs, + } + stmt, args, err = sqlx.Named(insCmdStmt, namedArgs) + if err != nil { + return ctxerr.Wrap(ctx, err, "prepare insert nano commands") + } + stmt, args, err = sqlx.In(stmt, args...) + if err != nil { + return ctxerr.Wrap(ctx, err, "expand IN arguments to insert nano commands") + } + if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "insert nano commands") + } + + // enqueue the nano command in the nano queue + stmt, args, err = sqlx.In(insNanoQueueStmt, hostUUID, hostID, execIDs) + if err != nil { + return ctxerr.Wrap(ctx, err, "prepare insert nano queue") + } + if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "insert nano queue") + } + + // best-effort APNs push notification to the host, not critical because we + // have a cron job that will retry for hosts with pending MDM commands. + if ds.pusher != nil { + if _, err := ds.pusher.Push(ctx, []string{hostUUID}); err != nil { + level.Error(ds.logger).Log("msg", "failed to send push notification", "err", err, "hostID", hostID, "hostUUID", hostUUID) //nolint:errcheck + } + } + return nil +} diff --git a/server/datastore/mysql/activities_test.go b/server/datastore/mysql/activities_test.go index d44d548a15cb..b14f77abce17 100644 --- a/server/datastore/mysql/activities_test.go +++ b/server/datastore/mysql/activities_test.go @@ -13,6 +13,8 @@ import ( "github.com/fleetdm/fleet/v4/pkg/scripts" "github.com/fleetdm/fleet/v4/server/contexts/viewer" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/mdm" + nanomdm_mysql "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/storage/mysql" "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/test" "github.com/google/uuid" @@ -37,6 +39,8 @@ func TestActivity(t *testing.T) { {"ListHostPastActivities", testListHostPastActivities}, {"CleanupActivitiesAndAssociatedData", testCleanupActivitiesAndAssociatedData}, {"CleanupActivitiesAndAssociatedDataBatch", testCleanupActivitiesAndAssociatedDataBatch}, + {"ActivateNextActivity", testActivateNextActivity}, + {"ActivateItselfOnEmptyQueue", testActivateItselfOnEmptyQueue}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -473,31 +477,24 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { require.NoError(t, err) // install the VPP app on h1 - commander, _ := createMDMAppleCommanderAndStorage(t, ds) - err = ds.InsertHostVPPSoftwareInstall(ctx, h1.ID, vppApp.VPPAppID, vppCommand1, "event-id-1", false, nil) - require.NoError(t, err) - err = commander.EnqueueCommand( - ctx, - []string{h1.UUID}, - createRawAppleCmd("InstallApplication", vppCommand1), - ) + err = ds.InsertHostVPPSoftwareInstall(ctx, h1.ID, vppApp.VPPAppID, vppCommand1, "event-id-1", fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) + // vppCommand1 is now active for h1 + // install the VPP app on h2, self-service - err = ds.InsertHostVPPSoftwareInstall(noUserCtx, h2.ID, vppApp.VPPAppID, vppCommand2, "event-id-2", true, nil) - require.NoError(t, err) - err = commander.EnqueueCommand( - ctx, - []string{h1.UUID}, - createRawAppleCmd("InstallApplication", vppCommand2), - ) + err = ds.InsertHostVPPSoftwareInstall(noUserCtx, h2.ID, vppApp.VPPAppID, vppCommand2, "event-id-2", fleet.HostSoftwareInstallOptions{SelfService: true}) require.NoError(t, err) + // vppCommand2 is now active for h2 - // create a sync script request for h1 that has been pending for > MaxWaitTime, will not show up + // create a sync script request for h1 that has been pending for > + // MaxWaitTime, will still show up (sync scripts go through the upcoming + // queue as any script) hsr, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h1.ID, ScriptContents: "sync", UserID: &u.ID, SyncRequest: true}) require.NoError(t, err) hSyncExpired := hsr.ExecutionID + t.Log("hSyncExpired", hSyncExpired) ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - _, err := q.ExecContext(ctx, "UPDATE host_script_results SET created_at = ? WHERE execution_id = ?", time.Now().Add(-(scripts.MaxServerWaitTime + time.Minute)), hSyncExpired) + _, err := q.ExecContext(ctx, "UPDATE upcoming_activities SET created_at = ? WHERE execution_id = ?", time.Now().Add(-(scripts.MaxServerWaitTime + time.Minute)), hSyncExpired) return err }) @@ -505,38 +502,32 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h1.ID, ScriptID: &scr1.ID, ScriptContents: scr1.ScriptContents, UserID: &u.ID}) require.NoError(t, err) h1A := hsr.ExecutionID + t.Log("h1A", h1A) + hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h1.ID, ScriptID: &scr2.ID, ScriptContents: scr2.ScriptContents, UserID: &u.ID}) require.NoError(t, err) h1B := hsr.ExecutionID + t.Log("h1B", h1B) + hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h1.ID, ScriptContents: "C", UserID: &u.ID}) require.NoError(t, err) h1C := hsr.ExecutionID + t.Log("h1C", h1C) + hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h1.ID, ScriptContents: "D"}) require.NoError(t, err) h1D := hsr.ExecutionID + t.Log("h1D", h1D) + hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h1.ID, ScriptContents: "E"}) require.NoError(t, err) h1E := hsr.ExecutionID - // create some software installs requests for h1, make some complete - h1FooFailed, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.InstallerID, false, nil) - require.NoError(t, err) - h1Bar, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw2Meta.InstallerID, false, nil) - require.NoError(t, err) - err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ - HostID: h1.ID, - InstallUUID: h1FooFailed, - PreInstallConditionOutput: ptr.String(""), // pre-install failed - }) - require.NoError(t, err) - h1FooInstalled, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.InstallerID, false, nil) - require.NoError(t, err) - err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ - HostID: h1.ID, - InstallUUID: h1FooInstalled, - PreInstallConditionOutput: ptr.String("ok"), - InstallScriptExitCode: ptr.Int(0), - }) + t.Log("h1E", h1E) + + // create some software installs requests for h1 + h1Bar, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw2Meta.InstallerID, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) + t.Log("h1Bar", h1Bar) // No user for this one and not Self-service, means it was installed by Fleet policy, err := ds.NewTeamPolicy(ctx, 0, &u.ID, fleet.PolicyPayload{ @@ -544,25 +535,23 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { Query: "SELECT 1", }) require.NoError(t, err) - h1Fleet, err := ds.InsertSoftwareInstallRequest(noUserCtx, h1.ID, sw1Meta.InstallerID, false, &policy.ID) + h1Fleet, err := ds.InsertSoftwareInstallRequest(noUserCtx, h1.ID, sw1Meta.InstallerID, fleet.HostSoftwareInstallOptions{PolicyID: &policy.ID}) require.NoError(t, err) + t.Log("h1Fleet", h1Fleet) - // create a single pending request for h2, as well as a non-pending one + // create a single pending request for h2 hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h2.ID, ScriptID: &scr1.ID, ScriptContents: scr1.ScriptContents, UserID: &u.ID}) require.NoError(t, err) h2A := hsr.ExecutionID - hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h2.ID, ScriptContents: "F", UserID: &u.ID}) - require.NoError(t, err) - _, _, err = ds.SetHostScriptExecutionResult(ctx, - &fleet.HostScriptResultPayload{HostID: h2.ID, ExecutionID: hsr.ExecutionID, Output: "ok", ExitCode: 0}) - require.NoError(t, err) - h2F := hsr.ExecutionID + t.Log("h2A", h2A) // add a pending software install request for h2 - h2Bar, err := ds.InsertSoftwareInstallRequest(ctx, h2.ID, sw2Meta.InstallerID, false, nil) + h2Bar, err := ds.InsertSoftwareInstallRequest(ctx, h2.ID, sw2Meta.InstallerID, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) + t.Log("h2Bar", h2Bar) // No user for this one and Self-service, means it was installed by the end user, so the user_id should be null/nil. - h2SelfService, err := ds.InsertSoftwareInstallRequest(noUserCtx, h2.ID, sw1Meta.InstallerID, true, nil) + h2SelfService, err := ds.InsertSoftwareInstallRequest(noUserCtx, h2.ID, sw1Meta.InstallerID, fleet.HostSoftwareInstallOptions{SelfService: true}) require.NoError(t, err) + t.Log("h2SelfService", h2SelfService) setupExpScript := &fleet.Script{Name: "setup_experience_script", ScriptContents: "setup_experience"} err = ds.SetSetupExperienceScript(ctx, setupExpScript) @@ -572,9 +561,10 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h2.ID, ScriptContents: "setup_experience", SetupExperienceScriptID: &ses.ID}) require.NoError(t, err) h2SetupExp := hsr.ExecutionID + t.Log("h2SetupExp", h2SetupExp) // create pending install and uninstall requests for h3 that will be deleted - _, err = ds.InsertSoftwareInstallRequest(ctx, h3.ID, sw3Meta.InstallerID, false, nil) + _, err = ds.InsertSoftwareInstallRequest(ctx, h3.ID, sw3Meta.InstallerID, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) err = ds.InsertSoftwareUninstallRequest(ctx, "uninstallRun", h3.ID, sw3Meta.InstallerID) require.NoError(t, err) @@ -590,7 +580,7 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { require.NoError(t, err) // h4A := hsr.ExecutionID // h4Bar, err := ds.InsertSoftwareInstallRequest(ctx, h4.ID, sw2Meta.InstallerID, false, nil) - _, err = ds.InsertSoftwareInstallRequest(ctx, h4.ID, sw2Meta.InstallerID, false, nil) + _, err = ds.InsertSoftwareInstallRequest(ctx, h4.ID, sw2Meta.InstallerID, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) // Delete the host err = ds.DeleteHost(ctx, h4.ID) @@ -600,25 +590,19 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { require.NoError(t, err) // force-set the order of the created_at timestamps - endTime := SetOrderedCreatedAtTimestamps(t, ds, time.Now(), "host_script_results", "execution_id", h1A, h1B) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h1FooFailed, h1Bar) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_script_results", "execution_id", h1C, h1D, h1E) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h1FooInstalled, h1Fleet) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h1Fleet) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h2SelfService) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h2Bar) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_script_results", "execution_id", h2A, h2F) - endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_vpp_software_installs", "command_uuid", vppCommand1, vppCommand2) - SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_script_results", "execution_id", h2SetupExp) + // even if vppCommand1 and 2 are later, since they are already activated + // (because they were enqueued first) they will show up first. + SetOrderedCreatedAtTimestamps(t, ds, time.Now(), "upcoming_activities", "execution_id", + h1A, h1B, h1Bar, h1C, h1D, h1E, h1Fleet, h2SelfService, h2Bar, h2A, vppCommand1, vppCommand2, h2SetupExp) execIDsWithUser := map[string]bool{ + hSyncExpired: true, h1A: true, h1B: true, h1C: true, h1D: false, h1E: false, h2A: true, - h2F: true, h1Fleet: false, h2SelfService: false, h1Bar: true, @@ -642,6 +626,11 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { execIDsFromPolicyAutomation := map[string]struct{}{ h1Fleet: {}, } + // to simplify map, false = cancellable, true = NON-cancellable + execIDsNonCancellable := map[string]bool{ + vppCommand1: true, + vppCommand2: true, + } cases := []struct { opts fleet.ListOptions @@ -652,49 +641,61 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { { opts: fleet.ListOptions{PerPage: 2}, hostID: h1.ID, - wantExecs: []string{h1A, h1B}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false, TotalResults: 8}, + wantExecs: []string{vppCommand1, hSyncExpired}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false, TotalResults: 9}, }, { opts: fleet.ListOptions{Page: 1, PerPage: 2}, hostID: h1.ID, - wantExecs: []string{h1Bar, h1C}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true, TotalResults: 8}, + wantExecs: []string{h1A, h1B}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true, TotalResults: 9}, }, { opts: fleet.ListOptions{Page: 2, PerPage: 2}, hostID: h1.ID, - wantExecs: []string{h1D, h1E}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true, TotalResults: 8}, + wantExecs: []string{h1Bar, h1C}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true, TotalResults: 9}, }, { opts: fleet.ListOptions{Page: 3, PerPage: 2}, hostID: h1.ID, - wantExecs: []string{h1Fleet, vppCommand1}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 8}, + wantExecs: []string{h1D, h1E}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true, TotalResults: 9}, + }, + { + opts: fleet.ListOptions{Page: 4, PerPage: 2}, + hostID: h1.ID, + wantExecs: []string{h1Fleet}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 9}, }, { opts: fleet.ListOptions{PerPage: 4}, hostID: h1.ID, - wantExecs: []string{h1A, h1B, h1Bar, h1C}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false, TotalResults: 8}, + wantExecs: []string{vppCommand1, hSyncExpired, h1A, h1B}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false, TotalResults: 9}, }, { opts: fleet.ListOptions{Page: 1, PerPage: 4}, hostID: h1.ID, - wantExecs: []string{h1D, h1E, h1Fleet, vppCommand1}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 8}, + wantExecs: []string{h1Bar, h1C, h1D, h1E}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true, TotalResults: 9}, }, { opts: fleet.ListOptions{Page: 2, PerPage: 4}, hostID: h1.ID, + wantExecs: []string{h1Fleet}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 9}, + }, + { + opts: fleet.ListOptions{Page: 3, PerPage: 4}, + hostID: h1.ID, wantExecs: []string{}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 8}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 9}, }, { opts: fleet.ListOptions{PerPage: 5}, hostID: h2.ID, - wantExecs: []string{h2SelfService, h2Bar, h2A, vppCommand2, h2SetupExp}, + wantExecs: []string{vppCommand2, h2SetupExp, h2SelfService, h2Bar, h2A}, // setup experience is top-priority, but vppCommand2 was already activated wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: false, TotalResults: 5}, }, { @@ -714,7 +715,7 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { t.Run(fmt.Sprintf("%v: %#v", c.hostID, c.opts), func(t *testing.T) { // always include metadata c.opts.IncludeMetadata = true - c.opts.OrderKey = "created_at" + c.opts.OrderKey = "" c.opts.OrderDirection = fleet.OrderAscending acts, meta, err := ds.ListHostUpcomingActivities(ctx, c.hostID, c.opts) @@ -754,6 +755,8 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { t.Fatalf("unknown activity type %s", a.Type) } + require.Equal(t, !execIDsNonCancellable[wantExec], a.Cancellable, "result %d", i) + if _, ok := execIDsFromPolicyAutomation[wantExec]; ok { require.Nil(t, a.ActorID, "result %d", i) require.NotNil(t, a.ActorFullName, "result %d", i) @@ -772,10 +775,14 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { require.Equal(t, wantUser.Email, *a.ActorEmail, "result %d", i) } else { require.Nil(t, a.ActorID, "result %d", i) - require.Nil(t, a.ActorFullName, "result %d", i) + if a.FleetInitiated { + require.NotNil(t, a.ActorFullName, "result %d", i) + require.Equal(t, "Fleet", *a.ActorFullName, "result %d", i) + } else { + require.Nil(t, a.ActorFullName, "result %d", i) + } require.Nil(t, a.ActorEmail, "result %d", i) } - } }) } @@ -1118,3 +1125,427 @@ func testCleanupActivitiesAndAssociatedDataBatch(t *testing.T, ds *Datastore) { }) require.Equal(t, 250, queriesLen) } + +func testActivateNextActivity(t *testing.T, ds *Datastore) { + ctx := context.Background() + ctx = context.WithValue(ctx, fleet.ActivityWebhookContextKey, true) + + test.CreateInsertGlobalVPPToken(t, ds) + + h1 := test.NewHost(t, ds, "h1.local", "10.10.10.1", "1", "1", time.Now()) + nanoEnrollAndSetHostMDMData(t, ds, h1, false) + h2 := test.NewHost(t, ds, "h2.local", "10.10.10.2", "2", "2", time.Now()) + nanoEnrollAndSetHostMDMData(t, ds, h2, false) + + u := test.NewUser(t, ds, "user1", "user1@example.com", false) + + nanoDB, err := nanomdm_mysql.New(nanomdm_mysql.WithDB(ds.primary.DB)) + require.NoError(t, err) + nanoCtx := &mdm.Request{EnrollID: &mdm.EnrollID{ID: h1.UUID}, Context: ctx} + + // create a couple VPP apps that can be installed later + vppApp1 := &fleet.VPPApp{ + Name: "vpp_1", VPPAppTeam: fleet.VPPAppTeam{VPPAppID: fleet.VPPAppID{AdamID: "vpp1", Platform: fleet.MacOSPlatform}}, + BundleIdentifier: "vpp1", + } + _, err = ds.InsertVPPAppWithTeam(ctx, vppApp1, nil) + require.NoError(t, err) + vppApp2 := &fleet.VPPApp{ + Name: "vpp_2", VPPAppTeam: fleet.VPPAppTeam{VPPAppID: fleet.VPPAppID{AdamID: "vpp2", Platform: fleet.MacOSPlatform}}, + BundleIdentifier: "vpp2", + } + _, err = ds.InsertVPPAppWithTeam(ctx, vppApp2, nil) + require.NoError(t, err) + + // create a software installer that can be installed later + installer1, err := fleet.NewTempFileReader(strings.NewReader("echo"), t.TempDir) + require.NoError(t, err) + sw1, _, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "install foo", + InstallerFile: installer1, + StorageID: uuid.NewString(), + Filename: "foo.pkg", + Title: "foo", + Source: "apps", + Version: "0.0.1", + UserID: u.ID, + UninstallScript: "uninstall foo", + ValidatedLabels: &fleet.LabelIdentsWithScope{}, + }) + require.NoError(t, err) + + // activating an empty queue is fine, nothing activated + execIDs, err := ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), h1.ID, "") + require.NoError(t, err) + require.Empty(t, execIDs) + + // activating when empty with an unknown completed exec id is fine + execIDs, err = ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), h1.ID, uuid.NewString()) + require.NoError(t, err) + require.Empty(t, execIDs) + + // create a script execution request + hsr, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ + HostID: h1.ID, + ScriptContents: "echo 'a'", + }) + require.NoError(t, err) + script1_1 := hsr.ExecutionID + + // create a second script execution request that will not be activated yet + hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ + HostID: h1.ID, + ScriptContents: "echo 'b'", + }) + require.NoError(t, err) + script1_2 := hsr.ExecutionID + + // add a couple install requests for vpp1 and vpp2 + vpp1_1 := uuid.NewString() + err = ds.InsertHostVPPSoftwareInstall(ctx, h1.ID, vppApp1.VPPAppID, vpp1_1, "event-id-1", fleet.HostSoftwareInstallOptions{}) + require.NoError(t, err) + vpp1_2 := uuid.NewString() + err = ds.InsertHostVPPSoftwareInstall(ctx, h1.ID, vppApp2.VPPAppID, vpp1_2, "event-id-2", fleet.HostSoftwareInstallOptions{}) + require.NoError(t, err) + + // activating does nothing because the first script is still activated + execIDs, err = ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), h1.ID, "") + require.NoError(t, err) + require.Empty(t, execIDs) + + // pending activities are script1_1, script1_2, vpp1_1, vpp1_2 + pendingActs, _, err := ds.ListHostUpcomingActivities(ctx, h1.ID, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, pendingActs, 4) + require.Equal(t, script1_1, pendingActs[0].UUID) + require.False(t, pendingActs[0].Cancellable) + require.Equal(t, script1_2, pendingActs[1].UUID) + require.True(t, pendingActs[1].Cancellable) + require.Equal(t, vpp1_1, pendingActs[2].UUID) + require.True(t, pendingActs[2].Cancellable) + require.Equal(t, vpp1_2, pendingActs[3].UUID) + require.True(t, pendingActs[3].Cancellable) + + // listing scripts ready to execute returns script1_1 + pendingScripts, err := ds.ListReadyToExecuteScriptsForHost(ctx, h1.ID, false) + require.NoError(t, err) + require.Len(t, pendingScripts, 1) + require.Equal(t, script1_1, pendingScripts[0].ExecutionID) + + // get host script result while there are no results yet returns the current status + scriptRes, err := ds.GetHostScriptExecutionResult(ctx, script1_1) + require.NoError(t, err) + require.Nil(t, scriptRes.ExitCode) + + scriptRes, err = ds.GetHostScriptExecutionResult(ctx, script1_2) + require.NoError(t, err) + require.Nil(t, scriptRes.ExitCode) + + // delete the script1_2 upcoming activity as if it was cancelled + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, ` + DELETE FROM upcoming_activities + WHERE execution_id = ?`, + script1_2) + return err + }) + + // set a script result, will activate both VPP apps + _, _, err = ds.SetHostScriptExecutionResult(ctx, &fleet.HostScriptResultPayload{ + HostID: h1.ID, ExecutionID: script1_1, Output: "a", ExitCode: 0, + }) + require.NoError(t, err) + + // get host script result now returns the result + scriptRes, err = ds.GetHostScriptExecutionResult(ctx, script1_1) + require.NoError(t, err) + require.NotNil(t, scriptRes.ExitCode) + require.EqualValues(t, 0, *scriptRes.ExitCode) + + // pending activities are vpp1_1, vpp1_2, both are non-cancellable because activated + pendingActs, _, err = ds.ListHostUpcomingActivities(ctx, h1.ID, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, pendingActs, 2) + require.Equal(t, vpp1_1, pendingActs[0].UUID) + require.False(t, pendingActs[0].Cancellable) + require.Equal(t, vpp1_2, pendingActs[1].UUID) + require.False(t, pendingActs[1].Cancellable) + + // nano commands have been inserted + cmd, err := nanoDB.RetrieveNextCommand(nanoCtx, false) + require.NoError(t, err) + require.Equal(t, vpp1_1, cmd.CommandUUID) + require.Equal(t, "InstallApplication", cmd.Command.Command.RequestType) + rawCmd := string(cmd.Raw) + require.Contains(t, rawCmd, ">"+vppApp1.VPPAppTeam.AdamID+"<") + require.Contains(t, rawCmd, ">"+vpp1_1+"<") + + // insert a result for that command and create the past activity, + // which triggers the next activity to be activated (should be none + // in this scenario, as one is still active) + cmdRes := &mdm.CommandResults{ + CommandUUID: vpp1_1, + Status: "Acknowledged", + Raw: []byte(``), + } + err = nanoDB.StoreCommandReport(nanoCtx, cmdRes) + require.NoError(t, err) + + err = ds.NewActivity(ctx, nil, fleet.ActivityInstalledAppStoreApp{ + HostID: h1.ID, + AppStoreID: vppApp1.VPPAppTeam.AdamID, + CommandUUID: vpp1_1, + }, []byte(`{}`), time.Now()) + require.NoError(t, err) + + appleCmdRes, err := ds.GetMDMAppleCommandResults(ctx, vpp1_1) + require.NoError(t, err) + require.Len(t, appleCmdRes, 1) + require.Equal(t, "Acknowledged", appleCmdRes[0].Status) + + pendingActs, _, err = ds.ListHostUpcomingActivities(ctx, h1.ID, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, pendingActs, 1) + require.Equal(t, vpp1_2, pendingActs[0].UUID) + require.False(t, pendingActs[0].Cancellable) + + // vpp1_2 is now the next nano command + cmd, err = nanoDB.RetrieveNextCommand(nanoCtx, false) + require.NoError(t, err) + require.Equal(t, vpp1_2, cmd.CommandUUID) + require.Equal(t, "InstallApplication", cmd.Command.Command.RequestType) + rawCmd = string(cmd.Raw) + require.Contains(t, rawCmd, ">"+vppApp2.VPPAppTeam.AdamID+"<") + require.Contains(t, rawCmd, ">"+vpp1_2+"<") + + // create a pending software install request + sw1_1, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1, fleet.HostSoftwareInstallOptions{}) + require.NoError(t, err) + + // the software install request is not active yet, so with only active, returns nothing + pendingSw, err := ds.ListReadyToExecuteSoftwareInstalls(ctx, h1.ID) + require.NoError(t, err) + require.Len(t, pendingSw, 0) + + // without only active, returns it + pendingSw, err = ds.ListPendingSoftwareInstalls(ctx, h1.ID) + require.NoError(t, err) + require.Len(t, pendingSw, 1) + require.Equal(t, sw1_1, pendingSw[0]) + + // activating does nothing because the VPP app 2 is still activated + execIDs, err = ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), h1.ID, "") + require.NoError(t, err) + require.Empty(t, execIDs) + + // trying to activate from a non-activated execution id (here, the software + // install sw1_1 one) does not delete that activity - it deletes only if it + // was activated + execIDs, err = ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), h1.ID, sw1_1) + require.NoError(t, err) + require.Empty(t, execIDs) + + pendingActs, _, err = ds.ListHostUpcomingActivities(ctx, h1.ID, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, pendingActs, 2) + require.Equal(t, vpp1_2, pendingActs[0].UUID) + require.False(t, pendingActs[0].Cancellable) + require.Equal(t, sw1_1, pendingActs[1].UUID) + require.True(t, pendingActs[1].Cancellable) + + // create a pending uninstall request + sw1_2 := uuid.NewString() + err = ds.InsertSoftwareUninstallRequest(ctx, sw1_2, h1.ID, sw1) + require.NoError(t, err) + + // still hasn't changed the pending queue + pendingActs, _, err = ds.ListHostUpcomingActivities(ctx, h1.ID, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, pendingActs, 3) + require.Equal(t, vpp1_2, pendingActs[0].UUID) + require.False(t, pendingActs[0].Cancellable) + require.Equal(t, sw1_1, pendingActs[1].UUID) + require.True(t, pendingActs[1].Cancellable) + require.Equal(t, sw1_2, pendingActs[2].UUID) + require.True(t, pendingActs[2].Cancellable) + + // insert a result for the vpp1_2 command + cmdRes = &mdm.CommandResults{ + CommandUUID: vpp1_2, + Status: "Error", + Raw: []byte(``), + } + err = nanoDB.StoreCommandReport(nanoCtx, cmdRes) + require.NoError(t, err) + + err = ds.NewActivity(ctx, nil, fleet.ActivityInstalledAppStoreApp{ + HostID: h1.ID, + AppStoreID: vppApp2.VPPAppTeam.AdamID, + CommandUUID: vpp1_2, + }, []byte(`{}`), time.Now()) + require.NoError(t, err) + + appleCmdRes, err = ds.GetMDMAppleCommandResults(ctx, vpp1_2) + require.NoError(t, err) + require.Len(t, appleCmdRes, 1) + require.Equal(t, "Error", appleCmdRes[0].Status) + + // software install activity is now activated + pendingActs, _, err = ds.ListHostUpcomingActivities(ctx, h1.ID, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, pendingActs, 2) + require.Equal(t, sw1_1, pendingActs[0].UUID) + require.False(t, pendingActs[0].Cancellable) + require.Equal(t, sw1_2, pendingActs[1].UUID) + require.True(t, pendingActs[1].Cancellable) + + // set a result for the software install + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: h1.ID, + InstallUUID: sw1_1, + InstallScriptExitCode: ptr.Int(0), + }) + require.NoError(t, err) + + swRes, err := ds.GetSoftwareInstallResults(ctx, sw1_1) + require.NoError(t, err) + require.Equal(t, fleet.SoftwareInstalled, swRes.Status) + + // activating does nothing because the sw1_2 was automatically activated + execIDs, err = ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), h1.ID, sw1_1) + require.NoError(t, err) + require.Empty(t, execIDs) + + pendingActs, _, err = ds.ListHostUpcomingActivities(ctx, h1.ID, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, pendingActs, 1) + require.Equal(t, sw1_2, pendingActs[0].UUID) + require.False(t, pendingActs[0].Cancellable) + + // set a result for the software uninstall + _, _, err = ds.SetHostScriptExecutionResult(ctx, &fleet.HostScriptResultPayload{ + HostID: h1.ID, + ExecutionID: sw1_2, + ExitCode: 1, + }) + require.NoError(t, err) + + // because the install and uninstall are for the same software installer, + // only the latest attempt is shown in the summary and it is the uninstall. + swSummary, err := ds.GetSummaryHostSoftwareInstalls(ctx, sw1) + require.NoError(t, err) + require.Equal(t, fleet.SoftwareInstallerStatusSummary{ + FailedUninstall: 1, + }, *swSummary) + + // activating does nothing because the queue is now empty + execIDs, err = ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), h1.ID, sw1_2) + require.NoError(t, err) + require.Empty(t, execIDs) + + pendingActs, _, err = ds.ListHostUpcomingActivities(ctx, h1.ID, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, pendingActs, 0) +} + +func testActivateItselfOnEmptyQueue(t *testing.T, ds *Datastore) { + ctx := context.Background() + ctx = context.WithValue(ctx, fleet.ActivityWebhookContextKey, true) + test.CreateInsertGlobalVPPToken(t, ds) + + h1 := test.NewHost(t, ds, "h1.local", "10.10.10.1", "1", "1", time.Now()) + nanoEnrollAndSetHostMDMData(t, ds, h1, false) + u := test.NewUser(t, ds, "user1", "user1@example.com", false) + + nanoDB, err := nanomdm_mysql.New(nanomdm_mysql.WithDB(ds.primary.DB)) + require.NoError(t, err) + nanoCtx := &mdm.Request{EnrollID: &mdm.EnrollID{ID: h1.UUID}, Context: ctx} + + vppApp1 := &fleet.VPPApp{ + Name: "vpp_1", VPPAppTeam: fleet.VPPAppTeam{VPPAppID: fleet.VPPAppID{AdamID: "vpp1", Platform: fleet.MacOSPlatform}}, + BundleIdentifier: "vpp1", + } + _, err = ds.InsertVPPAppWithTeam(ctx, vppApp1, nil) + require.NoError(t, err) + + installer1, err := fleet.NewTempFileReader(strings.NewReader("echo"), t.TempDir) + require.NoError(t, err) + sw1, _, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "install foo", + InstallerFile: installer1, + StorageID: uuid.NewString(), + Filename: "foo.pkg", + Title: "foo", + Source: "apps", + Version: "0.0.1", + UserID: u.ID, + UninstallScript: "uninstall foo", + ValidatedLabels: &fleet.LabelIdentsWithScope{}, + }) + require.NoError(t, err) + + // create a pending software install request + sw1_1, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1, fleet.HostSoftwareInstallOptions{}) + require.NoError(t, err) + + // set a result for the software install + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: h1.ID, + InstallUUID: sw1_1, + InstallScriptExitCode: ptr.Int(0), + }) + require.NoError(t, err) + + // create a pending script execution request + hsr, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ + HostID: h1.ID, + ScriptContents: "echo 'a'", + }) + require.NoError(t, err) + script1_1 := hsr.ExecutionID + + // set a result for the script + _, _, err = ds.SetHostScriptExecutionResult(ctx, &fleet.HostScriptResultPayload{ + HostID: h1.ID, ExecutionID: script1_1, Output: "a", ExitCode: 0, + }) + require.NoError(t, err) + + // create a pending uninstall request + sw1_2 := uuid.NewString() + err = ds.InsertSoftwareUninstallRequest(ctx, sw1_2, h1.ID, sw1) + require.NoError(t, err) + + // set a result for the software uninstall + _, _, err = ds.SetHostScriptExecutionResult(ctx, &fleet.HostScriptResultPayload{ + HostID: h1.ID, + ExecutionID: sw1_2, + ExitCode: 1, + }) + require.NoError(t, err) + + // create a pending vpp app install + vpp1_1 := uuid.NewString() + err = ds.InsertHostVPPSoftwareInstall(ctx, h1.ID, vppApp1.VPPAppID, vpp1_1, "event-id-1", fleet.HostSoftwareInstallOptions{}) + require.NoError(t, err) + + // set the result for the vpp app + cmdRes := &mdm.CommandResults{ + CommandUUID: vpp1_1, + Status: "Error", + Raw: []byte(``), + } + err = nanoDB.StoreCommandReport(nanoCtx, cmdRes) + require.NoError(t, err) + err = ds.NewActivity(ctx, nil, fleet.ActivityInstalledAppStoreApp{ + HostID: h1.ID, + AppStoreID: vppApp1.VPPAppTeam.AdamID, + CommandUUID: vpp1_1, + }, []byte(`{}`), time.Now()) + require.NoError(t, err) + + // the upcoming queue should be empty, each result having emptied the list + // and each enqueue having triggered the next activity. + pendingActs, _, err := ds.ListHostUpcomingActivities(ctx, h1.ID, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, pendingActs, 0) +} diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 35318813b68a..c8b8565b5470 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -548,6 +548,7 @@ var hostRefs = []string{ "host_activities", "host_mdm_actions", "host_calendar_events", + "upcoming_activities", } // NOTE: The following tables are explicity excluded from hostRefs list and accordingly are not @@ -1097,7 +1098,7 @@ func (ds *Datastore) applyHostFilters( // software (version) ID filter is mutually exclusive with software title ID // so we're reusing the same filter to avoid adding unnecessary conditions. if opt.SoftwareStatusFilter != nil { - meta, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, opt.TeamFilter, *opt.SoftwareTitleIDFilter, false) + _, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, opt.TeamFilter, *opt.SoftwareTitleIDFilter, false) switch { case fleet.IsNotFound(err): vppApp, err := ds.GetVPPAppByTeamAndTitleID(ctx, opt.TeamFilter, *opt.SoftwareTitleIDFilter) @@ -1114,7 +1115,9 @@ func (ds *Datastore) applyHostFilters( case err != nil: return "", nil, ctxerr.Wrap(ctx, err, "get software installer metadata by team and title id") default: - installerJoin, installerParams, err := ds.softwareInstallerJoin(meta.InstallerID, *opt.SoftwareStatusFilter) + // TODO(sarah): prior code was joining on installer id but based on how list options are parsed [1] it seems like this should be the title id + // [1] https://github.com/fleetdm/fleet/blob/8aecae4d853829cb6e7f828099a4f0953643cf18/server/datastore/mysql/hosts.go#L1088-L1089 + installerJoin, installerParams, err := ds.softwareInstallerJoin(*opt.SoftwareTitleIDFilter, *opt.SoftwareStatusFilter) if err != nil { return "", nil, ctxerr.Wrap(ctx, err, "software installer join") } diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index f08c37bf5489..edb0fa529bc3 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -6966,7 +6966,19 @@ func testHostsDeleteHosts(t *testing.T, ds *Datastore) { require.NoError(t, err) err = ds.RecordHostBootstrapPackage(context.Background(), "command-uuid", host.UUID) require.NoError(t, err) - _, err = ds.NewHostScriptExecutionRequest(context.Background(), &fleet.HostScriptRequestPayload{HostID: host.ID, ScriptContents: "foo"}) + + // this will create the row in both upcoming_activities and host_script_results as + // it will be activated immediately + hsr, err := ds.NewHostScriptExecutionRequest(context.Background(), &fleet.HostScriptRequestPayload{HostID: host.ID, ScriptContents: "foo"}) + require.NoError(t, err) + + // set a script result so it is removed from upcoming_activities and the + // software install (later in the test) can be inserted in both + // upcoming_activities and host_software_installs + _, _, err = ds.SetHostScriptExecutionResult(context.Background(), &fleet.HostScriptResultPayload{ + HostID: host.ID, + ExecutionID: hsr.ExecutionID, + }) require.NoError(t, err) _, err = ds.writer(context.Background()).Exec(` @@ -7030,7 +7042,7 @@ func testHostsDeleteHosts(t *testing.T, ds *Datastore) { ValidatedLabels: &fleet.LabelIdentsWithScope{}, }) require.NoError(t, err) - _, err = ds.InsertSoftwareInstallRequest(context.Background(), host.ID, softwareInstaller, false, nil) + _, err = ds.InsertSoftwareInstallRequest(context.Background(), host.ID, softwareInstaller, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) // Add an awaiting configuration entry diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index b420ba0917dd..c98c938bcc8c 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -673,17 +673,34 @@ func (ds *Datastore) applyHostLabelFilters(ctx context.Context, filter fleet.Tea // // TODO: Do we currently support filtering by software version ID and label? // } if opt.SoftwareTitleIDFilter != nil && opt.SoftwareStatusFilter != nil { - // get the installer id - meta, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, opt.TeamFilter, *opt.SoftwareTitleIDFilter, false) - if err != nil { + // check for software installer metadata + _, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, opt.TeamFilter, *opt.SoftwareTitleIDFilter, false) + switch { + case fleet.IsNotFound(err): + vppApp, err := ds.GetVPPAppByTeamAndTitleID(ctx, opt.TeamFilter, *opt.SoftwareTitleIDFilter) + if err != nil { + return "", nil, ctxerr.Wrap(ctx, err, "get vpp app by team and title id") + } + vppAppJoin, vppAppParams, err := ds.vppAppJoin(vppApp.VPPAppID, *opt.SoftwareStatusFilter) + if err != nil { + return "", nil, ctxerr.Wrap(ctx, err, "vpp app join") + } + softwareStatusJoin = vppAppJoin + joinParams = append(joinParams, vppAppParams...) + + case err != nil: return "", nil, ctxerr.Wrap(ctx, err, "get software installer metadata by team and title id") + + default: + // TODO(uniq): prior code was joining on installer id but based on how list options are parsed [1] it seems like this should be the title id + // [1] https://github.com/fleetdm/fleet/blob/8aecae4d853829cb6e7f828099a4f0953643cf18/server/datastore/mysql/hosts.go#L1088-L1089 + installerJoin, installerParams, err := ds.softwareInstallerJoin(*opt.SoftwareTitleIDFilter, *opt.SoftwareStatusFilter) + if err != nil { + return "", nil, ctxerr.Wrap(ctx, err, "software installer join") + } + softwareStatusJoin = installerJoin + joinParams = append(joinParams, installerParams...) } - installerJoin, installerParams, err := ds.softwareInstallerJoin(meta.InstallerID, *opt.SoftwareStatusFilter) - if err != nil { - return "", nil, ctxerr.Wrap(ctx, err, "software installer join") - } - softwareStatusJoin = installerJoin - joinParams = append(joinParams, installerParams...) } if softwareStatusJoin != "" { query += softwareStatusJoin diff --git a/server/datastore/mysql/migrations/tables/20241017163402_AddSoftwareDetailsToInstallRecords_test.go b/server/datastore/mysql/migrations/tables/20241017163402_AddSoftwareDetailsToInstallRecords_test.go index 0c58be339e6f..0c77ebbad0fe 100644 --- a/server/datastore/mysql/migrations/tables/20241017163402_AddSoftwareDetailsToInstallRecords_test.go +++ b/server/datastore/mysql/migrations/tables/20241017163402_AddSoftwareDetailsToInstallRecords_test.go @@ -119,7 +119,7 @@ func TestUp_20241017163402(t *testing.T) { require.Equal(t, "2024-10-01T00:00:00Z", result.UpdatedAt) // test activity hydration manual query - execNoErr(t, db, `INSERT INTO activities (activity_type, details) VALUES + execNoErr(t, db, `INSERT INTO activities (activity_type, details) VALUES ("installed_software", '{"install_uuid": "execution-id1", "software_title": "Foo", "software_package": "foo.pkg"}'), ("installed_software", '{"install_uuid": "execution-id2", "software_title": "A Real Title"}'), ("uninstalled_software", '{"execution_id": "execution-id3", "software_title": "Ignore Me"}')`) diff --git a/server/datastore/mysql/migrations/tables/20250127162751_AddUnifiedQueueTable.go b/server/datastore/mysql/migrations/tables/20250127162751_AddUnifiedQueueTable.go new file mode 100644 index 000000000000..fe60154839cb --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20250127162751_AddUnifiedQueueTable.go @@ -0,0 +1,163 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20250127162751, Down_20250127162751) +} + +func Up_20250127162751(tx *sql.Tx) error { + _, err := tx.Exec(` +CREATE TABLE upcoming_activities ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + host_id INT UNSIGNED NOT NULL, + + -- priority 0 is normal, > 0 is higher priority, < 0 is lower priority. + priority INT NOT NULL DEFAULT 0, + + -- user_id is the user that triggered the activity, it may be null if the + -- activity is fleet-initiated or the user was deleted. Additional user + -- information (name, email, etc.) is stored in the JSON payload. + user_id INT UNSIGNED NULL, + fleet_initiated TINYINT(1) NOT NULL DEFAULT 0, + + -- type of activity to be executed, currently we only support those, but as + -- more activity types get added, we can enrich the ENUM with an ALTER TABLE. + activity_type ENUM('script', 'software_install', 'software_uninstall', 'vpp_app_install') NOT NULL, + + -- execution_id is the identifier of the activity that will be used when + -- executed - e.g. scripts and software installs have an execution_id, and + -- it is sometimes important to know it as soon as the activity is enqueued, + -- so we need to generate it immediately. Every activity will be identified + -- via this unique execution_id. + execution_id VARCHAR(255) NOT NULL, + payload JSON NOT NULL, + + -- Using DATETIME instead of TIMESTAMP to prevent future Y2K38 issues + activated_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT NOW(6), + updated_at DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6), + + PRIMARY KEY (id), + UNIQUE KEY idx_upcoming_activities_execution_id (execution_id), + -- index for the common access pattern to get the next activity to execute + INDEX idx_upcoming_activities_host_id_priority_created_at (host_id, priority, created_at), + -- index for the common access pattern to get by activity type (e.g. deleting pending scripts) + INDEX idx_upcoming_activities_host_id_activity_type (activity_type, host_id), + CONSTRAINT fk_upcoming_activities_user_id + FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE SET NULL +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci +`, + ) + if err != nil { + return fmt.Errorf("failed to create upcoming_activities: %w", err) + } + + _, err = tx.Exec(` +CREATE TABLE script_upcoming_activities ( + upcoming_activity_id BIGINT UNSIGNED NOT NULL, + + -- those are all columns and not JSON fields because we need FKs on them to + -- do processing ON DELETE, otherwise we'd have to check for existence of + -- each one when executing the activity (we need the enqueue next activity + -- action to be efficient). + script_id INT UNSIGNED NULL, + script_content_id INT UNSIGNED NULL, + policy_id INT UNSIGNED NULL, + setup_experience_script_id INT UNSIGNED NULL, + + -- Using DATETIME instead of TIMESTAMP to prevent future Y2K38 issues + created_at DATETIME(6) NOT NULL DEFAULT NOW(6), + updated_at DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6), + + PRIMARY KEY (upcoming_activity_id), + CONSTRAINT fk_script_upcoming_activities_upcoming_activity_id + FOREIGN KEY (upcoming_activity_id) REFERENCES upcoming_activities (id) ON DELETE CASCADE, + CONSTRAINT fk_script_upcoming_activities_script_id + FOREIGN KEY (script_id) REFERENCES scripts (id) ON DELETE SET NULL, + CONSTRAINT fk_script_upcoming_activities_script_content_id + FOREIGN KEY (script_content_id) REFERENCES script_contents (id) ON DELETE CASCADE, + CONSTRAINT fk_script_upcoming_activities_policy_id + FOREIGN KEY (policy_id) REFERENCES policies (id) ON DELETE SET NULL, + CONSTRAINT fk_script_upcoming_activities_setup_experience_script_id + FOREIGN KEY (setup_experience_script_id) REFERENCES setup_experience_scripts (id) ON DELETE SET NULL +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci +`, + ) + if err != nil { + return err + } + + _, err = tx.Exec(` +CREATE TABLE software_install_upcoming_activities ( + upcoming_activity_id BIGINT UNSIGNED NOT NULL, + + -- those are all columns and not JSON fields because we need FKs on them to + -- do processing ON DELETE, otherwise we'd have to check for existence of + -- each one when executing the activity (we need the enqueue next activity + -- action to be efficient). + software_installer_id INT UNSIGNED NULL, + policy_id INT UNSIGNED NULL, + software_title_id INT UNSIGNED NULL, + + -- Using DATETIME instead of TIMESTAMP to prevent future Y2K38 issues + created_at DATETIME(6) NOT NULL DEFAULT NOW(6), + updated_at DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6), + + PRIMARY KEY (upcoming_activity_id), + CONSTRAINT fk_software_install_upcoming_activities_upcoming_activity_id + FOREIGN KEY (upcoming_activity_id) REFERENCES upcoming_activities (id) ON DELETE CASCADE, + CONSTRAINT fk_software_install_upcoming_activities_software_installer_id + FOREIGN KEY (software_installer_id) REFERENCES software_installers (id) ON DELETE SET NULL ON UPDATE CASCADE, + CONSTRAINT fk_software_install_upcoming_activities_policy_id + FOREIGN KEY (policy_id) REFERENCES policies (id) ON DELETE SET NULL, + CONSTRAINT fk_software_install_upcoming_activities_software_title_id + FOREIGN KEY (software_title_id) REFERENCES software_titles (id) ON DELETE SET NULL ON UPDATE CASCADE +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci +`, + ) + if err != nil { + return err + } + + _, err = tx.Exec(` +CREATE TABLE vpp_app_upcoming_activities ( + upcoming_activity_id BIGINT UNSIGNED NOT NULL, + + -- those are all columns and not JSON fields because we need FKs on them to + -- do processing ON DELETE, otherwise we'd have to check for existence of + -- each one when executing the activity (we need the enqueue next activity + -- action to be efficient). + adam_id VARCHAR(16) NOT NULL, + platform VARCHAR(10) NOT NULL, + vpp_token_id INT UNSIGNED NULL, + policy_id INT UNSIGNED NULL, + + -- Using DATETIME instead of TIMESTAMP to prevent future Y2K38 issues + created_at DATETIME(6) NOT NULL DEFAULT NOW(6), + updated_at DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6), + + PRIMARY KEY (upcoming_activity_id), + CONSTRAINT fk_vpp_app_upcoming_activities_upcoming_activity_id + FOREIGN KEY (upcoming_activity_id) REFERENCES upcoming_activities (id) ON DELETE CASCADE, + CONSTRAINT fk_vpp_app_upcoming_activities_adam_id_platform + FOREIGN KEY (adam_id, platform) REFERENCES vpp_apps (adam_id, platform) ON DELETE CASCADE, + CONSTRAINT fk_vpp_app_upcoming_activities_vpp_token_id + FOREIGN KEY (vpp_token_id) REFERENCES vpp_tokens (id) ON DELETE SET NULL, + CONSTRAINT fk_vpp_app_upcoming_activities_policy_id + FOREIGN KEY (policy_id) REFERENCES policies (id) ON DELETE SET NULL +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci +`, + ) + if err != nil { + return err + } + return nil +} + +func Down_20250127162751(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/mysql.go b/server/datastore/mysql/mysql.go index d5d0c0372f84..8c36d1116919 100644 --- a/server/datastore/mysql/mysql.go +++ b/server/datastore/mysql/mysql.go @@ -26,6 +26,7 @@ import ( "github.com/fleetdm/fleet/v4/server/datastore/mysql/migrations/tables" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/goose" + nano_push "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/push" scep_depot "github.com/fleetdm/fleet/v4/server/mdm/scep/depot" "github.com/go-kit/log" "github.com/go-kit/log/level" @@ -53,6 +54,7 @@ type Datastore struct { logger log.Logger clock clock.Clock config config.MysqlConfig + pusher nano_push.Pusher // nil if no read replica readReplicaConfig *config.MysqlConfig @@ -97,11 +99,23 @@ type Datastore struct { // e.g.: testBatchSetMDMWindowsProfilesErr = "insert:fail" testBatchSetMDMWindowsProfilesErr string + // set this to the execution ids of activities that should be activated in + // the next call to activateNextUpcomingActivity, instead of picking the next + // available activity based on normal prioritization and creation date + // ordering. + testActivateSpecificNextActivities []string + // This key is used to encrypt sensitive data stored in the Fleet DB, for example MDM // certificates and keys. serverPrivateKey string } +// WithPusher sets an APNs pusher for the datastore, used when activating +// next activities that require MDM commands. +func (ds *Datastore) WithPusher(p nano_push.Pusher) { + ds.pusher = p +} + // reader returns the DB instance to use for read-only statements, which is the // replica unless the primary has been explicitly required via // ctxdb.RequirePrimary. diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go index 9421a706b5a1..b8f763a9cd03 100644 --- a/server/datastore/mysql/policies_test.go +++ b/server/datastore/mysql/policies_test.go @@ -308,7 +308,7 @@ func testGlobalPolicyPendingScriptsAndInstalls(t *testing.T, ds *Datastore) { require.Len(t, policies, 1) // create a pending software install request - _, err = ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID, false, &policy2.ID) + _, err = ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID, fleet.HostSoftwareInstallOptions{PolicyID: &policy2.ID}) require.NoError(t, err) pendingInstalls, err := ds.ListPendingSoftwareInstalls(ctx, host2.ID) @@ -900,7 +900,7 @@ func testTeamPolicyPendingScriptsAndInstalls(t *testing.T, ds *Datastore) { }) require.NoError(t, err) // create a pending software install request - _, err = ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID, false, &policy2.ID) + _, err = ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID, fleet.HostSoftwareInstallOptions{PolicyID: &policy2.ID}) require.NoError(t, err) pendingInstalls, err := ds.ListPendingSoftwareInstalls(ctx, host2.ID) diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 9aa007a4d312..87cdf5a1c9a5 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -1128,9 +1128,9 @@ CREATE TABLE `migration_status_tables` ( `is_applied` tinyint(1) NOT NULL, `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=350 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=351 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'),(350,20250127162751,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `mobile_device_management_solutions` ( @@ -1664,6 +1664,28 @@ CREATE TABLE `script_contents` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `script_upcoming_activities` ( + `upcoming_activity_id` bigint unsigned NOT NULL, + `script_id` int unsigned DEFAULT NULL, + `script_content_id` int unsigned DEFAULT NULL, + `policy_id` int unsigned DEFAULT NULL, + `setup_experience_script_id` int unsigned DEFAULT NULL, + `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + PRIMARY KEY (`upcoming_activity_id`), + KEY `fk_script_upcoming_activities_script_id` (`script_id`), + KEY `fk_script_upcoming_activities_script_content_id` (`script_content_id`), + KEY `fk_script_upcoming_activities_policy_id` (`policy_id`), + KEY `fk_script_upcoming_activities_setup_experience_script_id` (`setup_experience_script_id`), + CONSTRAINT `fk_script_upcoming_activities_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `policies` (`id`) ON DELETE SET NULL, + CONSTRAINT `fk_script_upcoming_activities_script_content_id` FOREIGN KEY (`script_content_id`) REFERENCES `script_contents` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_script_upcoming_activities_script_id` FOREIGN KEY (`script_id`) REFERENCES `scripts` (`id`) ON DELETE SET NULL, + CONSTRAINT `fk_script_upcoming_activities_setup_experience_script_id` FOREIGN KEY (`setup_experience_script_id`) REFERENCES `setup_experience_scripts` (`id`) ON DELETE SET NULL, + CONSTRAINT `fk_script_upcoming_activities_upcoming_activity_id` FOREIGN KEY (`upcoming_activity_id`) REFERENCES `upcoming_activities` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `scripts` ( `id` int unsigned NOT NULL AUTO_INCREMENT, `team_id` int unsigned DEFAULT NULL, @@ -1817,6 +1839,25 @@ CREATE TABLE `software_host_counts` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `software_install_upcoming_activities` ( + `upcoming_activity_id` bigint unsigned NOT NULL, + `software_installer_id` int unsigned DEFAULT NULL, + `policy_id` int unsigned DEFAULT NULL, + `software_title_id` int unsigned DEFAULT NULL, + `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + PRIMARY KEY (`upcoming_activity_id`), + KEY `fk_software_install_upcoming_activities_software_installer_id` (`software_installer_id`), + KEY `fk_software_install_upcoming_activities_policy_id` (`policy_id`), + KEY `fk_software_install_upcoming_activities_software_title_id` (`software_title_id`), + CONSTRAINT `fk_software_install_upcoming_activities_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `policies` (`id`) ON DELETE SET NULL, + CONSTRAINT `fk_software_install_upcoming_activities_software_installer_id` FOREIGN KEY (`software_installer_id`) REFERENCES `software_installers` (`id`) ON DELETE SET NULL ON UPDATE CASCADE, + CONSTRAINT `fk_software_install_upcoming_activities_software_title_id` FOREIGN KEY (`software_title_id`) REFERENCES `software_titles` (`id`) ON DELETE SET NULL ON UPDATE CASCADE, + CONSTRAINT `fk_software_install_upcoming_activities_upcoming_activity_id` FOREIGN KEY (`upcoming_activity_id`) REFERENCES `upcoming_activities` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `software_installer_labels` ( `id` int unsigned NOT NULL AUTO_INCREMENT, `software_installer_id` int unsigned NOT NULL, @@ -1931,6 +1972,28 @@ CREATE TABLE `teams` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `upcoming_activities` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `host_id` int unsigned NOT NULL, + `priority` int NOT NULL DEFAULT '0', + `user_id` int unsigned DEFAULT NULL, + `fleet_initiated` tinyint(1) NOT NULL DEFAULT '0', + `activity_type` enum('script','software_install','software_uninstall','vpp_app_install') COLLATE utf8mb4_unicode_ci NOT NULL, + `execution_id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + `payload` json NOT NULL, + `activated_at` datetime(6) DEFAULT NULL, + `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + PRIMARY KEY (`id`), + UNIQUE KEY `idx_upcoming_activities_execution_id` (`execution_id`), + KEY `idx_upcoming_activities_host_id_priority_created_at` (`host_id`,`priority`,`created_at`), + KEY `idx_upcoming_activities_host_id_activity_type` (`activity_type`,`host_id`), + KEY `fk_upcoming_activities_user_id` (`user_id`), + CONSTRAINT `fk_upcoming_activities_user_id` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`) ON DELETE SET NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `user_teams` ( `user_id` int unsigned NOT NULL, `team_id` int unsigned NOT NULL, @@ -1994,6 +2057,26 @@ CREATE TABLE `vpp_app_team_labels` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `vpp_app_upcoming_activities` ( + `upcoming_activity_id` bigint unsigned NOT NULL, + `adam_id` varchar(16) COLLATE utf8mb4_unicode_ci NOT NULL, + `platform` varchar(10) COLLATE utf8mb4_unicode_ci NOT NULL, + `vpp_token_id` int unsigned DEFAULT NULL, + `policy_id` int unsigned DEFAULT NULL, + `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + PRIMARY KEY (`upcoming_activity_id`), + KEY `fk_vpp_app_upcoming_activities_adam_id_platform` (`adam_id`,`platform`), + KEY `fk_vpp_app_upcoming_activities_vpp_token_id` (`vpp_token_id`), + KEY `fk_vpp_app_upcoming_activities_policy_id` (`policy_id`), + CONSTRAINT `fk_vpp_app_upcoming_activities_adam_id_platform` FOREIGN KEY (`adam_id`, `platform`) REFERENCES `vpp_apps` (`adam_id`, `platform`) ON DELETE CASCADE, + CONSTRAINT `fk_vpp_app_upcoming_activities_policy_id` FOREIGN KEY (`policy_id`) REFERENCES `policies` (`id`) ON DELETE SET NULL, + CONSTRAINT `fk_vpp_app_upcoming_activities_upcoming_activity_id` FOREIGN KEY (`upcoming_activity_id`) REFERENCES `upcoming_activities` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_vpp_app_upcoming_activities_vpp_token_id` FOREIGN KEY (`vpp_token_id`) REFERENCES `vpp_tokens` (`id`) ON DELETE SET NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `vpp_apps` ( `adam_id` varchar(16) COLLATE utf8mb4_unicode_ci NOT NULL, `title_id` int unsigned DEFAULT NULL, diff --git a/server/datastore/mysql/scripts.go b/server/datastore/mysql/scripts.go index c0c5280823b2..c0ad3ab0b184 100644 --- a/server/datastore/mysql/scripts.go +++ b/server/datastore/mysql/scripts.go @@ -14,19 +14,11 @@ import ( constants "github.com/fleetdm/fleet/v4/pkg/scripts" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/go-kit/log/level" "github.com/google/uuid" "github.com/jmoiron/sqlx" ) -const whereFilterPendingScript = ` - exit_code IS NULL - -- async requests + sync requests created within the given interval - AND ( - sync_request = 0 - OR created_at >= DATE_SUB(NOW(), INTERVAL ? SECOND) - ) -` - func (ds *Datastore) NewHostScriptExecutionRequest(ctx context.Context, request *fleet.HostScriptRequestPayload) (*fleet.HostScriptResult, error) { var res *fleet.HostScriptResult return res, ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { @@ -41,52 +33,84 @@ func (ds *Datastore) NewHostScriptExecutionRequest(ctx context.Context, request id, _ := scRes.LastInsertId() request.ScriptContentID = uint(id) //nolint:gosec // dismiss G115 } - res, err = newHostScriptExecutionRequest(ctx, tx, request, false) - return err - }) -} - -func (ds *Datastore) NewInternalScriptExecutionRequest(ctx context.Context, request *fleet.HostScriptRequestPayload) (*fleet.HostScriptResult, error) { - var res *fleet.HostScriptResult - var err error - return res, ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - if request.ScriptContentID == 0 { - return errors.New("script contents must be saved prior to execution") - } - res, err = newHostScriptExecutionRequest(ctx, tx, request, true) + res, err = ds.newHostScriptExecutionRequest(ctx, tx, request, false) return err }) } -func newHostScriptExecutionRequest(ctx context.Context, tx sqlx.ExtContext, request *fleet.HostScriptRequestPayload, isInternal bool) (*fleet.HostScriptResult, error) { +func (ds *Datastore) newHostScriptExecutionRequest(ctx context.Context, tx sqlx.ExtContext, request *fleet.HostScriptRequestPayload, isInternal bool) (*fleet.HostScriptResult, error) { const ( - insStmt = `INSERT INTO host_script_results (host_id, execution_id, script_content_id, output, script_id, policy_id, user_id, sync_request, setup_experience_script_id, is_internal) VALUES (?, ?, ?, '', ?, ?, ?, ?, ?, ?)` - getStmt = `SELECT hsr.id, hsr.host_id, hsr.execution_id, hsr.created_at, hsr.script_id, hsr.policy_id, hsr.user_id, hsr.sync_request, sc.contents as script_contents, hsr.setup_experience_script_id FROM host_script_results hsr JOIN script_contents sc WHERE sc.id = hsr.script_content_id AND hsr.id = ?` + insUAStmt = ` +INSERT INTO upcoming_activities + (host_id, priority, user_id, fleet_initiated, activity_type, execution_id, payload) +VALUES + (?, ?, ?, ?, 'script', ?, + JSON_OBJECT( + 'sync_request', ?, + 'is_internal', ?, + 'user', (SELECT JSON_OBJECT('name', name, 'email', email, 'gravatar_url', gravatar_url) FROM users WHERE id = ?) + ) + )` + + insSUAStmt = ` +INSERT INTO script_upcoming_activities + (upcoming_activity_id, script_id, script_content_id, policy_id, setup_experience_script_id) +VALUES + (?, ?, ?, ?, ?) +` + + getStmt = ` +SELECT + ua.id, ua.host_id, ua.execution_id, ua.created_at, sua.script_id, sua.policy_id, ua.user_id, + payload->'$.sync_request' AS sync_request, + sc.contents as script_contents, sua.setup_experience_script_id +FROM + upcoming_activities ua + INNER JOIN script_upcoming_activities sua + ON ua.id = sua.upcoming_activity_id + INNER JOIN script_contents sc + ON sua.script_content_id = sc.id +WHERE + ua.id = ? +` ) execID := uuid.New().String() - result, err := tx.ExecContext(ctx, insStmt, + result, err := tx.ExecContext(ctx, insUAStmt, request.HostID, + request.Priority(), + request.UserID, + request.PolicyID != nil, // fleet-initiated if request is via a policy failure execID, - request.ScriptContentID, + request.SyncRequest, + isInternal, + request.UserID, + ) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "new script upcoming activity") + } + + activityID, _ := result.LastInsertId() + _, err = tx.ExecContext(ctx, insSUAStmt, + activityID, request.ScriptID, + request.ScriptContentID, request.PolicyID, - request.UserID, - request.SyncRequest, request.SetupExperienceScriptID, - isInternal, ) if err != nil { - return nil, ctxerr.Wrap(ctx, err, "new host script execution request") + return nil, ctxerr.Wrap(ctx, err, "new join script upcoming activity") } var script fleet.HostScriptResult - id, _ := result.LastInsertId() - err = sqlx.GetContext(ctx, tx, &script, getStmt, id) + err = sqlx.GetContext(ctx, tx, &script, getStmt, activityID) if err != nil { - return nil, ctxerr.Wrap(ctx, err, "getting the created host script result to return") + return nil, ctxerr.Wrap(ctx, err, "getting the created host script activity to return") } + if _, err := ds.activateNextUpcomingActivity(ctx, tx, request.HostID, ""); err != nil { + return nil, ctxerr.Wrap(ctx, err, "activate next activity") + } return &script, nil } @@ -159,6 +183,17 @@ func (ds *Datastore) SetHostScriptExecutionResult(ctx context.Context, result *f return ctxerr.Wrap(ctx, err, "check if host script result exists") } if resultExists { + level.Debug(ds.logger).Log("msg", "duplicate script execution result sent, will be ignored (original result is preserved)", + "host_id", result.HostID, + "execution_id", result.ExecutionID, + ) + + // still do the activate next activity to ensure progress as there was + // an unexpected flow if we get here. + if _, err := ds.activateNextUpcomingActivity(ctx, tx, result.HostID, result.ExecutionID); err != nil { + return ctxerr.Wrap(ctx, err, "activate next activity") + } + // succeed but leave hsr nil return nil } @@ -206,7 +241,7 @@ func (ds *Datastore) SetHostScriptExecutionResult(ctx context.Context, result *f case "": // do nothing case "uninstall": - err = updateUninstallStatusFromResult(ctx, tx, result.HostID, result.ExecutionID, result.ExitCode) + err = ds.updateUninstallStatusFromResult(ctx, tx, result.HostID, result.ExecutionID, result.ExitCode) if err != nil { return ctxerr.Wrap(ctx, err, "update host uninstall action based on script result") } @@ -217,6 +252,11 @@ func (ds *Datastore) SetHostScriptExecutionResult(ctx context.Context, result *f } } } + + if _, err := ds.activateNextUpcomingActivity(ctx, tx, result.HostID, result.ExecutionID); err != nil { + return ctxerr.Wrap(ctx, err, "activate next activity") + } + return nil }) if err != nil { @@ -226,26 +266,50 @@ func (ds *Datastore) SetHostScriptExecutionResult(ctx context.Context, result *f } func (ds *Datastore) ListPendingHostScriptExecutions(ctx context.Context, hostID uint, onlyShowInternal bool) ([]*fleet.HostScriptResult, error) { - internalWhere := "" + return ds.listUpcomingHostScriptExecutions(ctx, hostID, onlyShowInternal, false) +} + +func (ds *Datastore) ListReadyToExecuteScriptsForHost(ctx context.Context, hostID uint, onlyShowInternal bool) ([]*fleet.HostScriptResult, error) { + return ds.listUpcomingHostScriptExecutions(ctx, hostID, onlyShowInternal, true) +} + +func (ds *Datastore) listUpcomingHostScriptExecutions(ctx context.Context, hostID uint, onlyShowInternal, onlyReadyToExecute bool) ([]*fleet.HostScriptResult, error) { + extraWhere := "" if onlyShowInternal { - internalWhere = " AND is_internal = TRUE" + extraWhere = " AND ua.payload->'$.is_internal' = 1" + } + if onlyReadyToExecute { + extraWhere += " AND ua.activated_at IS NOT NULL" } - listStmt := fmt.Sprintf(` SELECT id, host_id, execution_id, - script_id - FROM - host_script_results - WHERE - host_id = ? AND - host_deleted_at IS NULL AND - %s - %s - ORDER BY - created_at ASC`, whereFilterPendingScript, internalWhere) + script_id, + created_at + FROM ( + SELECT + ua.id, + ua.host_id, + ua.execution_id, + sua.script_id, + ua.priority, + ua.created_at, + IF(ua.activated_at IS NULL, 0, 1) AS topmost + FROM + upcoming_activities ua + INNER JOIN script_upcoming_activities sua + ON ua.id = sua.upcoming_activity_id + WHERE + ua.host_id = ? AND + ua.activity_type = 'script' AND + ( + ua.payload->'$.sync_request' = 0 OR + ua.created_at >= DATE_SUB(NOW(), INTERVAL ? SECOND) + ) + %s + ORDER BY topmost DESC, priority DESC, created_at ASC) t`, extraWhere) var results []*fleet.HostScriptResult seconds := int(constants.MaxServerWaitTime.Seconds()) @@ -258,13 +322,15 @@ func (ds *Datastore) ListPendingHostScriptExecutions(ctx context.Context, hostID func (ds *Datastore) IsExecutionPendingForHost(ctx context.Context, hostID uint, scriptID uint) (bool, error) { const getStmt = ` SELECT - 1 + 1 FROM - host_script_results + upcoming_activities ua + INNER JOIN script_upcoming_activities sua + ON ua.id = sua.upcoming_activity_id WHERE - host_id = ? AND - script_id = ? AND - exit_code IS NULL + ua.host_id = ? AND + ua.activity_type = 'script' AND + sua.script_id = ? ` var results []*uint @@ -279,39 +345,73 @@ func (ds *Datastore) GetHostScriptExecutionResult(ctx context.Context, execID st } func (ds *Datastore) getHostScriptExecutionResultDB(ctx context.Context, q sqlx.QueryerContext, execID string) (*fleet.HostScriptResult, error) { - const getStmt = ` - SELECT - hsr.id, - hsr.host_id, - hsr.execution_id, - sc.contents as script_contents, - hsr.script_id, - hsr.policy_id, - hsr.output, - hsr.runtime, - hsr.exit_code, - hsr.timeout, - hsr.created_at, - hsr.user_id, - hsr.sync_request, - hsr.host_deleted_at, - hsr.setup_experience_script_id + const getActiveStmt = ` + SELECT + hsr.id, + hsr.host_id, + hsr.execution_id, + sc.contents as script_contents, + hsr.script_id, + hsr.policy_id, + hsr.output, + hsr.runtime, + hsr.exit_code, + hsr.timeout, + hsr.created_at, + hsr.user_id, + hsr.sync_request, + hsr.host_deleted_at, + hsr.setup_experience_script_id FROM - host_script_results hsr - JOIN - script_contents sc - WHERE - hsr.execution_id = ? - AND - hsr.script_content_id = sc.id + host_script_results hsr + JOIN + script_contents sc + WHERE + hsr.execution_id = ? AND + hsr.script_content_id = sc.id +` + + const getUpcomingStmt = ` + SELECT + 0 as id, + ua.host_id, + ua.execution_id, + sc.contents as script_contents, + sua.script_id, + sua.policy_id, + '' as output, + 0 as runtime, + NULL as exit_code, + NULL as timeout, + ua.created_at, + ua.user_id, + COALESCE(ua.payload->'$.sync_request', 0) as sync_request, + NULL as host_deleted_at, + sua.setup_experience_script_id + FROM + upcoming_activities ua + INNER JOIN script_upcoming_activities sua + ON ua.id = sua.upcoming_activity_id + INNER JOIN + script_contents sc + ON sua.script_content_id = sc.id + WHERE + ua.execution_id = ? AND + ua.activity_type = 'script' ` var result fleet.HostScriptResult - if err := sqlx.GetContext(ctx, q, &result, getStmt, execID); err != nil { - if err == sql.ErrNoRows { - return nil, ctxerr.Wrap(ctx, notFound("HostScriptResult").WithName(execID)) + if err := sqlx.GetContext(ctx, q, &result, getActiveStmt, execID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + // try with upcoming activities + err = sqlx.GetContext(ctx, q, &result, getUpcomingStmt, execID) + if errors.Is(err, sql.ErrNoRows) { + return nil, ctxerr.Wrap(ctx, notFound("HostScriptResult").WithName(execID)) + } + } + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "get host script result") } - return nil, ctxerr.Wrap(ctx, err, "get host script result") } return &result, nil } @@ -476,6 +576,21 @@ func (ds *Datastore) DeleteScript(ctx context.Context, id uint) error { return ctxerr.Wrapf(ctx, err, "cancel pending script executions") } + _, err = tx.ExecContext(ctx, `DELETE FROM upcoming_activities + USING upcoming_activities + INNER JOIN script_upcoming_activities sua + ON upcoming_activities.id = sua.upcoming_activity_id + WHERE sua.script_id = ? AND + upcoming_activities.activity_type = 'script' AND + (upcoming_activities.payload->'$.sync_request' = 0 OR + upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) + `, + id, int(constants.MaxServerWaitTime.Seconds()), + ) + if err != nil { + return ctxerr.Wrapf(ctx, err, "cancel upcoming pending script executions") + } + _, err = tx.ExecContext(ctx, `DELETE FROM scripts WHERE id = ?`, id) if err != nil { if isMySQLForeignKey(err) { @@ -502,7 +617,12 @@ func (ds *Datastore) deletePendingHostScriptExecutionsForPolicy(ctx context.Cont globalOrTeamID = *teamID } - deleteStmt := fmt.Sprintf(` + deletePendingFunc := func(stmt string, args ...any) error { + _, err := ds.writer(ctx).ExecContext(ctx, stmt, args...) + return ctxerr.Wrap(ctx, err, "delete pending host script executions for policy") + } + + deleteHSRStmt := ` DELETE FROM host_script_results WHERE @@ -510,13 +630,29 @@ func (ds *Datastore) deletePendingHostScriptExecutionsForPolicy(ctx context.Cont script_id IN ( SELECT id FROM scripts WHERE scripts.global_or_team_id = ? ) AND - %s - `, whereFilterPendingScript) + exit_code IS NULL + ` - seconds := int(constants.MaxServerWaitTime.Seconds()) - _, err := ds.writer(ctx).ExecContext(ctx, deleteStmt, policyID, globalOrTeamID, seconds) - if err != nil { - return ctxerr.Wrap(ctx, err, "delete pending host script executions for policy") + if err := deletePendingFunc(deleteHSRStmt, policyID, globalOrTeamID); err != nil { + return err + } + + deleteUAStmt := ` + DELETE FROM + upcoming_activities + USING + upcoming_activities + INNER JOIN script_upcoming_activities sua + ON upcoming_activities.id = sua.upcoming_activity_id + WHERE + upcoming_activities.activity_type = 'script' AND + sua.policy_id = ? AND + sua.script_id IN ( + SELECT id FROM scripts WHERE scripts.global_or_team_id = ? + ) +` + if err := deletePendingFunc(deleteUAStmt, policyID, globalOrTeamID); err != nil { + return err } return nil @@ -623,34 +759,73 @@ SELECT FROM scripts s LEFT JOIN ( + -- latest is in host_script_results only if none in upcoming_activities SELECT - id, - host_id, - script_id, - execution_id, - created_at, - exit_code + r.id, + r.host_id, + r.script_id, + r.execution_id, + r.created_at, + r.exit_code FROM host_script_results r + LEFT OUTER JOIN host_script_results r2 + ON r.host_id = r2.host_id AND + r.script_id = r2.script_id AND + (r2.created_at > r.created_at OR (r.created_at = r2.created_at AND r2.id > r.id)) WHERE - host_id = ? - AND NOT EXISTS ( - SELECT - 1 - FROM - host_script_results + r.host_id = ? AND + r2.id IS NULL AND -- no other row at a later time + NOT EXISTS ( + SELECT 1 + FROM upcoming_activities ua + INNER JOIN script_upcoming_activities sua + ON ua.id = sua.upcoming_activity_id WHERE - host_id = ? - AND id != r.id - AND script_id = r.script_id - AND(created_at > r.created_at - OR(created_at = r.created_at - AND id > r.id)))) hsr + ua.host_id = r.host_id AND + ua.activity_type = 'script' AND + sua.script_id = r.script_id + ) + + UNION + + -- latest is in upcoming_activities + SELECT + NULL as id, + ua.host_id, + sua.script_id, + ua.execution_id, + ua.created_at, + NULL as exit_code + FROM + upcoming_activities ua + INNER JOIN script_upcoming_activities sua + ON ua.id = sua.upcoming_activity_id + WHERE + ua.host_id = ? AND + ua.activity_type = 'script' AND + NOT EXISTS ( + -- no later entry in upcoming activities, not sure how + -- or if it can be done with the LEFT OUTER JOIN approach + -- because it involves 2 tables. + SELECT + 1 + FROM + upcoming_activities ua2 + INNER JOIN script_upcoming_activities sua2 + ON ua2.id = sua2.upcoming_activity_id + WHERE + ua.host_id = ua2.host_id AND + ua2.activity_type = 'script' AND + sua.script_id = sua2.script_id AND + (ua2.created_at > ua.created_at OR (ua.created_at = ua2.created_at AND ua2.id > ua.id)) + ) + ) hsr ON s.id = hsr.script_id WHERE (hsr.host_id IS NULL OR hsr.host_id = ?) AND s.global_or_team_id = ? - ` +` args := []any{hostID, hostID, hostID, globalOrTeamID} if len(extension) > 0 { @@ -701,9 +876,19 @@ WHERE ` const unsetAllScriptsFromPolicies = `UPDATE policies SET script_id = NULL WHERE team_id = ?` - const clearAllPendingExecutions = `DELETE FROM host_script_results WHERE - exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND) - AND script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ?)` + const clearAllPendingExecutionsHSR = `DELETE FROM host_script_results WHERE + exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND) + AND script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ?)` + + const clearAllPendingExecutionsUA = `DELETE FROM upcoming_activities + USING + upcoming_activities + INNER JOIN script_upcoming_activities sua + ON upcoming_activities.id = sua.upcoming_activity_id + WHERE + upcoming_activities.activity_type = 'script' + AND (upcoming_activities.payload->'$.sync_request' = 0 OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) + AND sua.script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ?)` const unsetScriptsNotInListFromPolicies = ` UPDATE policies SET script_id = NULL @@ -718,9 +903,19 @@ WHERE name NOT IN (?) ` - const clearPendingExecutionsNotInList = `DELETE FROM host_script_results WHERE - exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND) - AND script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ? AND name NOT IN (?))` + const clearPendingExecutionsNotInListHSR = `DELETE FROM host_script_results WHERE + exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND) + AND script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ? AND name NOT IN (?))` + + const clearPendingExecutionsNotInListUA = `DELETE FROM upcoming_activities + USING + upcoming_activities + INNER JOIN script_upcoming_activities sua + ON upcoming_activities.id = sua.upcoming_activity_id + WHERE + upcoming_activities.activity_type = 'script' + AND (upcoming_activities.payload->'$.sync_request' = 0 OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) + AND sua.script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ? AND name NOT IN (?))` const insertNewOrEditedScript = ` INSERT INTO @@ -733,9 +928,19 @@ ON DUPLICATE KEY UPDATE script_content_id = VALUES(script_content_id), id=LAST_INSERT_ID(id) ` - const clearPendingExecutionsWithObsoleteScript = `DELETE FROM host_script_results WHERE - exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND) - AND script_id = ? AND script_content_id != ?` + const clearPendingExecutionsWithObsoleteScriptHSR = `DELETE FROM host_script_results WHERE + exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND) + AND script_id = ? AND script_content_id != ?` + + const clearPendingExecutionsWithObsoleteScriptUA = `DELETE FROM upcoming_activities + USING + upcoming_activities + INNER JOIN script_upcoming_activities sua + ON upcoming_activities.id = sua.upcoming_activity_id + WHERE + upcoming_activities.activity_type = 'script' + AND (upcoming_activities.payload->'$.sync_request' = 0 OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) + AND sua.script_id = ? AND sua.script_content_id != ?` const loadInsertedScripts = `SELECT id, team_id, name FROM scripts WHERE global_or_team_id = ?` @@ -786,6 +991,8 @@ ON DUPLICATE KEY UPDATE policiesArgs []any executionsStmt string executionsArgs []any + extraExecStmt string + extraExecArgs []any err error ) if len(keepNames) > 0 { @@ -800,10 +1007,15 @@ ON DUPLICATE KEY UPDATE return ctxerr.Wrap(ctx, err, "build statement to unset obsolete scripts from policies") } - executionsStmt, executionsArgs, err = sqlx.In(clearPendingExecutionsNotInList, int(constants.MaxServerWaitTime.Seconds()), globalOrTeamID, keepNames) + executionsStmt, executionsArgs, err = sqlx.In(clearPendingExecutionsNotInListHSR, int(constants.MaxServerWaitTime.Seconds()), globalOrTeamID, keepNames) if err != nil { return ctxerr.Wrap(ctx, err, "build statement to clear pending script executions from obsolete scripts") } + + extraExecStmt, extraExecArgs, err = sqlx.In(clearPendingExecutionsNotInListUA, int(constants.MaxServerWaitTime.Seconds()), globalOrTeamID, keepNames) + if err != nil { + return ctxerr.Wrap(ctx, err, "build statement to clear upcoming pending script executions from obsolete scripts") + } } else { scriptsStmt = deleteAllScriptsInTeam scriptsArgs = []any{globalOrTeamID} @@ -811,8 +1023,11 @@ ON DUPLICATE KEY UPDATE policiesStmt = unsetAllScriptsFromPolicies policiesArgs = []any{globalOrTeamID} - executionsStmt = clearAllPendingExecutions + executionsStmt = clearAllPendingExecutionsHSR executionsArgs = []any{int(constants.MaxServerWaitTime.Seconds()), globalOrTeamID} + + extraExecStmt = clearAllPendingExecutionsUA + extraExecArgs = []any{int(constants.MaxServerWaitTime.Seconds()), globalOrTeamID} } if _, err := tx.ExecContext(ctx, policiesStmt, policiesArgs...); err != nil { return ctxerr.Wrap(ctx, err, "unset obsolete scripts from policies") @@ -820,6 +1035,9 @@ ON DUPLICATE KEY UPDATE if _, err := tx.ExecContext(ctx, executionsStmt, executionsArgs...); err != nil { return ctxerr.Wrap(ctx, err, "clear obsolete script pending executions") } + if _, err := tx.ExecContext(ctx, extraExecStmt, extraExecArgs...); err != nil { + return ctxerr.Wrap(ctx, err, "clear obsolete upcoming script pending executions") + } if _, err := tx.ExecContext(ctx, scriptsStmt, scriptsArgs...); err != nil { return ctxerr.Wrap(ctx, err, "delete obsolete scripts") } @@ -836,9 +1054,12 @@ ON DUPLICATE KEY UPDATE return ctxerr.Wrapf(ctx, err, "insert new/edited script with name %q", s.Name) } scriptID, _ := insertRes.LastInsertId() - if _, err := tx.ExecContext(ctx, clearPendingExecutionsWithObsoleteScript, int(constants.MaxServerWaitTime.Seconds()), scriptID, contentID); err != nil { + if _, err := tx.ExecContext(ctx, clearPendingExecutionsWithObsoleteScriptHSR, int(constants.MaxServerWaitTime.Seconds()), scriptID, contentID); err != nil { return ctxerr.Wrapf(ctx, err, "clear obsolete pending script executions with name %q", s.Name) } + if _, err := tx.ExecContext(ctx, clearPendingExecutionsWithObsoleteScriptUA, int(constants.MaxServerWaitTime.Seconds()), scriptID, contentID); err != nil { + return ctxerr.Wrapf(ctx, err, "clear obsolete upcoming pending script executions with name %q", s.Name) + } } if err := sqlx.SelectContext(ctx, tx, &insertedScripts, loadInsertedScripts, globalOrTeamID); err != nil { @@ -1048,7 +1269,7 @@ func (ds *Datastore) LockHostViaScript(ctx context.Context, request *fleet.HostS id, _ := scRes.LastInsertId() request.ScriptContentID = uint(id) //nolint:gosec // dismiss G115 - res, err = newHostScriptExecutionRequest(ctx, tx, request, true) + res, err = ds.newHostScriptExecutionRequest(ctx, tx, request, true) if err != nil { return ctxerr.Wrap(ctx, err, "lock host via script create execution") } @@ -1098,7 +1319,7 @@ func (ds *Datastore) UnlockHostViaScript(ctx context.Context, request *fleet.Hos id, _ := scRes.LastInsertId() request.ScriptContentID = uint(id) //nolint:gosec // dismiss G115 - res, err = newHostScriptExecutionRequest(ctx, tx, request, true) + res, err = ds.newHostScriptExecutionRequest(ctx, tx, request, true) if err != nil { return ctxerr.Wrap(ctx, err, "unlock host via script create execution") } @@ -1149,7 +1370,7 @@ func (ds *Datastore) WipeHostViaScript(ctx context.Context, request *fleet.HostS id, _ := scRes.LastInsertId() request.ScriptContentID = uint(id) //nolint:gosec // dismiss G115 - res, err = newHostScriptExecutionRequest(ctx, tx, request, true) + res, err = ds.newHostScriptExecutionRequest(ctx, tx, request, true) if err != nil { return ctxerr.Wrap(ctx, err, "wipe host via script create execution") } @@ -1283,13 +1504,16 @@ func updateHostLockWipeStatusFromResult(ctx context.Context, tx sqlx.ExtContext, return ctxerr.Wrap(ctx, err, "update host lock/wipe status from result") } -func updateUninstallStatusFromResult(ctx context.Context, tx sqlx.ExtContext, hostID uint, executionID string, exitCode int) error { +func (ds *Datastore) updateUninstallStatusFromResult(ctx context.Context, tx sqlx.ExtContext, hostID uint, executionID string, exitCode int) error { stmt := ` UPDATE host_software_installs SET uninstall_script_exit_code = ? WHERE execution_id = ? AND host_id = ? ` if _, err := tx.ExecContext(ctx, stmt, exitCode, executionID, hostID); err != nil { return ctxerr.Wrap(ctx, err, "update uninstall status from result") } + // NOTE: no need to call activateNextUpcomingActivity here as this function + // is called from SetHostScriptExecutionResult which will call it before + // completing. return nil } @@ -1313,6 +1537,9 @@ WHERE AND NOT EXISTS ( SELECT 1 FROM setup_experience_scripts WHERE script_content_id = script_contents.id ) + AND NOT EXISTS ( + SELECT 1 FROM script_upcoming_activities WHERE script_content_id = script_contents.id + ) ` _, err := ds.writer(ctx).ExecContext(ctx, deleteStmt) if err != nil { diff --git a/server/datastore/mysql/scripts_test.go b/server/datastore/mysql/scripts_test.go index 197732d04125..351c42400a89 100644 --- a/server/datastore/mysql/scripts_test.go +++ b/server/datastore/mysql/scripts_test.go @@ -65,29 +65,30 @@ func testHostScriptResult(t *testing.T, ds *Datastore) { // create a createdScript execution request (with a user) u := test.NewUser(t, ds, "Bob", "bob@example.com", true) - createdScript, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ + createdScript1, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ HostID: 1, ScriptContents: "echo", UserID: &u.ID, SyncRequest: true, }) require.NoError(t, err) - require.NotZero(t, createdScript.ID) - require.NotEmpty(t, createdScript.ExecutionID) - require.Equal(t, uint(1), createdScript.HostID) - require.NotEmpty(t, createdScript.ExecutionID) - require.Equal(t, "echo", createdScript.ScriptContents) - require.Nil(t, createdScript.ExitCode) - require.Empty(t, createdScript.Output) - require.NotNil(t, createdScript.UserID) - require.Equal(t, u.ID, *createdScript.UserID) - require.True(t, createdScript.SyncRequest) + require.NotZero(t, createdScript1.ID) + require.NotEmpty(t, createdScript1.ExecutionID) + require.Equal(t, uint(1), createdScript1.HostID) + require.NotEmpty(t, createdScript1.ExecutionID) + require.Equal(t, "echo", createdScript1.ScriptContents) + require.Nil(t, createdScript1.ExitCode) + require.Empty(t, createdScript1.Output) + require.NotNil(t, createdScript1.UserID) + require.Equal(t, u.ID, *createdScript1.UserID) + require.True(t, createdScript1.SyncRequest) + // createdScript1 is now activated, as the queue was empty // the script execution is now listed as pending for this host pending, err = ds.ListPendingHostScriptExecutions(ctx, 1, false) require.NoError(t, err) require.Len(t, pending, 1) - require.Equal(t, createdScript.ID, pending[0].ID) + require.Equal(t, createdScript1.ID, pending[0].ID) // the script execution isn't visible when looking at internal-only scripts pending, err = ds.ListPendingHostScriptExecutions(ctx, 1, true) @@ -97,7 +98,7 @@ func testHostScriptResult(t *testing.T, ds *Datastore) { // record a result for this execution hsr, action, err := ds.SetHostScriptExecutionResult(ctx, &fleet.HostScriptResultPayload{ HostID: 1, - ExecutionID: createdScript.ExecutionID, + ExecutionID: createdScript1.ExecutionID, Output: "foo", Runtime: 2, ExitCode: 0, @@ -110,7 +111,7 @@ func testHostScriptResult(t *testing.T, ds *Datastore) { // record a duplicate result for this execution, will be ignored hsr, _, err = ds.SetHostScriptExecutionResult(ctx, &fleet.HostScriptResultPayload{ HostID: 1, - ExecutionID: createdScript.ExecutionID, + ExecutionID: createdScript1.ExecutionID, Output: "foobarbaz", Runtime: 22, ExitCode: 1, @@ -125,30 +126,41 @@ func testHostScriptResult(t *testing.T, ds *Datastore) { require.Empty(t, pending) // the script result can be retrieved - script, err := ds.GetHostScriptExecutionResult(ctx, createdScript.ExecutionID) + script, err := ds.GetHostScriptExecutionResult(ctx, createdScript1.ExecutionID) require.NoError(t, err) - expectScript := *createdScript + expectScript := *createdScript1 expectScript.Output = "foo" expectScript.Runtime = 2 expectScript.ExitCode = ptr.Int64(0) expectScript.Timeout = ptr.Int(300) + expectScript.CreatedAt, script.CreatedAt = time.Time{}, time.Time{} require.Equal(t, &expectScript, script) // create another script execution request (null user id this time) - createdScript, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ + time.Sleep(time.Millisecond) // ensure a different timestamp + createdScript2, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ HostID: 1, ScriptContents: "echo2", }) require.NoError(t, err) - require.NotZero(t, createdScript.ID) - require.NotEmpty(t, createdScript.ExecutionID) - require.Nil(t, createdScript.UserID) - require.False(t, createdScript.SyncRequest) + require.NotZero(t, createdScript2.ID) + require.NotEmpty(t, createdScript2.ExecutionID) + require.Nil(t, createdScript2.UserID) + require.False(t, createdScript2.SyncRequest) + // createdScript2 is now activated as the queue was empty + + // the script execution is now listed as pending for this host + pending, err = ds.ListPendingHostScriptExecutions(ctx, 1, false) + require.NoError(t, err) + require.Len(t, pending, 1) + require.Equal(t, createdScript2.ID, pending[0].ID) // the script result can be retrieved even if it has no result yet - script, err = ds.GetHostScriptExecutionResult(ctx, createdScript.ExecutionID) + script, err = ds.GetHostScriptExecutionResult(ctx, createdScript2.ExecutionID) require.NoError(t, err) - require.Equal(t, createdScript, script) + expectedScript := *createdScript2 + expectedScript.CreatedAt, script.CreatedAt = time.Time{}, time.Time{} + require.Equal(t, &expectedScript, script) // record a result for this execution, with an output that is too large largeOutput := strings.Repeat("a", 1000) + @@ -162,6 +174,7 @@ func testHostScriptResult(t *testing.T, ds *Datastore) { strings.Repeat("i", 1000) + strings.Repeat("j", 1000) + strings.Repeat("k", 1000) + // Note that the expectation is that the "a"s get truncated expectedOutput := strings.Repeat("b", 1000) + strings.Repeat("c", 1000) + strings.Repeat("d", 1000) + @@ -175,7 +188,7 @@ func testHostScriptResult(t *testing.T, ds *Datastore) { _, _, err = ds.SetHostScriptExecutionResult(ctx, &fleet.HostScriptResultPayload{ HostID: 1, - ExecutionID: createdScript.ExecutionID, + ExecutionID: createdScript2.ExecutionID, Output: largeOutput, Runtime: 10, ExitCode: 1, @@ -184,59 +197,49 @@ func testHostScriptResult(t *testing.T, ds *Datastore) { require.NoError(t, err) // the script result can be retrieved - script, err = ds.GetHostScriptExecutionResult(ctx, createdScript.ExecutionID) + script, err = ds.GetHostScriptExecutionResult(ctx, createdScript2.ExecutionID) require.NoError(t, err) require.Equal(t, expectedOutput, script.Output) // create an async execution request - createdScript, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ + time.Sleep(time.Millisecond) // ensure a different timestamp + createdScript3, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ HostID: 1, ScriptContents: "echo 3", UserID: &u.ID, SyncRequest: false, }) require.NoError(t, err) - require.NotZero(t, createdScript.ID) - require.NotEmpty(t, createdScript.ExecutionID) - require.Equal(t, uint(1), createdScript.HostID) - require.NotEmpty(t, createdScript.ExecutionID) - require.Equal(t, "echo 3", createdScript.ScriptContents) - require.Nil(t, createdScript.ExitCode) - require.Empty(t, createdScript.Output) - require.NotNil(t, createdScript.UserID) - require.Equal(t, u.ID, *createdScript.UserID) - require.False(t, createdScript.SyncRequest) + require.NotZero(t, createdScript3.ID) + require.NotEmpty(t, createdScript3.ExecutionID) + require.Equal(t, uint(1), createdScript3.HostID) + require.NotEmpty(t, createdScript3.ExecutionID) + require.Equal(t, "echo 3", createdScript3.ScriptContents) + require.Nil(t, createdScript3.ExitCode) + require.Empty(t, createdScript3.Output) + require.NotNil(t, createdScript3.UserID) + require.Equal(t, u.ID, *createdScript3.UserID) + require.False(t, createdScript3.SyncRequest) + // createdScript3 is now activated as the queue was empty // the script execution is now listed as pending for this host pending, err = ds.ListPendingHostScriptExecutions(ctx, 1, false) require.NoError(t, err) require.Len(t, pending, 1) - require.Equal(t, createdScript.ID, pending[0].ID) + require.Equal(t, createdScript3.ID, pending[0].ID) - // modify the timestamp of the script to simulate an script that has + // modify the upcoming script to be a sync script that has // been pending for a long time ExecAdhocSQL(t, ds, func(tx sqlx.ExtContext) error { - _, err := tx.ExecContext(ctx, "UPDATE host_script_results SET created_at = ? WHERE id = ?", time.Now().Add(-24*time.Hour), createdScript.ID) - return err - }) - - // the script execution still shows as pending - pending, err = ds.ListPendingHostScriptExecutions(ctx, 1, false) - require.NoError(t, err) - require.Len(t, pending, 1) - require.Equal(t, createdScript.ID, pending[0].ID) - - // modify the script to be a sync script that has - // been pending for a long time - ExecAdhocSQL(t, ds, func(tx sqlx.ExtContext) error { - _, err := tx.ExecContext(ctx, "UPDATE host_script_results SET sync_request = 1 WHERE id = ?", createdScript.ID) + _, err := tx.ExecContext(ctx, "UPDATE upcoming_activities SET created_at = ?, payload = JSON_SET(payload, '$.sync_request', true) WHERE id = ?", + time.Now().Add(-24*time.Hour), createdScript3.ID) return err }) // the script is not pending anymore pending, err = ds.ListPendingHostScriptExecutions(ctx, 1, false) require.NoError(t, err) - require.Empty(t, pending, 0) + require.Len(t, pending, 0) // check that scripts with large unsigned error codes get // converted to signed error codes @@ -248,6 +251,18 @@ func testHostScriptResult(t *testing.T, ds *Datastore) { }) require.NoError(t, err) + // record a result for createdScript3 so that the unsigned script gets activated + _, _, err = ds.SetHostScriptExecutionResult(ctx, &fleet.HostScriptResultPayload{ + HostID: 1, + ExecutionID: createdScript3.ExecutionID, + Output: "foo", + Runtime: 1, + ExitCode: 0, + Timeout: 0, + }) + require.NoError(t, err) + // createdUnsignedScript is now activated, record its result + unsignedScriptResult, _, err := ds.SetHostScriptExecutionResult(ctx, &fleet.HostScriptResultPayload{ HostID: 1, ExecutionID: createdUnsignedScript.ExecutionID, @@ -528,6 +543,7 @@ func testListScripts(t *testing.T, ds *Datastore) { func testGetHostScriptDetails(t *testing.T, ds *Datastore) { ctx := context.Background() + t.Cleanup(func() { ds.testActivateSpecificNextActivities = nil }) names := []string{"script-1.sh", "script-2.sh", "script-3.sh", "script-4.sh", "script-5.sh"} for _, r := range append(names[1:], names[0]) { @@ -550,29 +566,54 @@ func testGetHostScriptDetails(t *testing.T, ds *Datastore) { require.Len(t, scripts, 6) insertResults := func(t *testing.T, hostID uint, script *fleet.Script, createdAt time.Time, execID string, exitCode *int64) { - stmt := ` -INSERT INTO - host_script_results (%s host_id, created_at, execution_id, exit_code, output) -VALUES - (%s ?,?,?,?,?)` - - args := []interface{}{} - if script.ID == 0 { - stmt = fmt.Sprintf(stmt, "", "") - } else { - stmt = fmt.Sprintf(stmt, "script_id,", "?,") - args = append(args, script.ID) + var scriptID *uint + if script.ID != 0 { + scriptID = &script.ID } - args = append(args, hostID, createdAt, execID, exitCode, "") - + hsr, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ + HostID: hostID, + ScriptID: scriptID, + }) + require.NoError(t, err) ExecAdhocSQL(t, ds, func(tx sqlx.ExtContext) error { - _, err := tx.ExecContext(ctx, stmt, args...) + _, err := tx.ExecContext(ctx, `UPDATE upcoming_activities SET execution_id = ?, created_at = ? WHERE execution_id = ?`, + execID, createdAt, hsr.ExecutionID) return err }) + if exitCode != nil { + ds.testActivateSpecificNextActivities = []string{execID} + act, err := ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), hostID, "") + require.NoError(t, err) + require.ElementsMatch(t, act, ds.testActivateSpecificNextActivities) + ds.testActivateSpecificNextActivities = nil + + _, _, err = ds.SetHostScriptExecutionResult(ctx, &fleet.HostScriptResultPayload{ + HostID: hostID, + ExecutionID: execID, + ExitCode: int(*exitCode), + }) + require.NoError(t, err) + + // force the test timestamp + ExecAdhocSQL(t, ds, func(tx sqlx.ExtContext) error { + _, err := tx.ExecContext(ctx, `UPDATE host_script_results SET created_at = ? WHERE execution_id = ?`, + createdAt, execID) + return err + }) + } } now := time.Now().UTC().Truncate(time.Second) + // add some results for an ad-hoc, non-saved script, should not be included in results + // create it first so that this one gets activated, and the other ones are never + // activated automatically. + _, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ + HostID: 42, + ScriptContents: "echo script-6", + }) + require.NoError(t, err) + // add some results for script-1 insertResults(t, 42, scripts[0], now.Add(-3*time.Minute), "execution-1-1", nil) insertResults(t, 42, scripts[0], now.Add(-1*time.Minute), "execution-1-2", nil) // last execution for script-1, status "pending" @@ -590,8 +631,9 @@ VALUES // add some results for script-4 insertResults(t, 42, scripts[3], now.Add(-1*time.Minute), "execution-4-1", ptr.Int64(-2)) // last execution for script-4, status "error" - // add some results for an ad-hoc, non-saved script, should not be included in results - insertResults(t, 42, &fleet.Script{Name: "script-6", ScriptContents: "echo script-6"}, now.Add(-1*time.Minute), "execution-6-1", ptr.Int64(0)) + // add a pending and a completed script execution for script-5 + insertResults(t, 42, scripts[4], now.Add(-2*time.Minute), "execution-5-1", ptr.Int64(0)) + insertResults(t, 42, scripts[4], now.Add(-3*time.Minute), "execution-5-2", nil) // upcoming is always latest, regardless of timestamp t.Run("results match expected formatting and filtering", func(t *testing.T) { res, _, err := ds.GetHostScriptDetails(ctx, 42, nil, fleet.ListOptions{}, "") @@ -625,7 +667,10 @@ VALUES require.Equal(t, "error", r.LastExecution.Status) case scripts[4].ID: require.Equal(t, scripts[4].Name, r.Name) - require.Nil(t, r.LastExecution) + require.NotNil(t, r.LastExecution) + // require.Equal(t, now.Add(-3*time.Minute), r.LastExecution.ExecutedAt) + require.Equal(t, "execution-5-2", r.LastExecution.ExecutionID) + require.Equal(t, "pending", r.LastExecution.Status) case scripts[5].ID: require.Equal(t, scripts[5].Name, r.Name) require.Nil(t, r.LastExecution) @@ -727,29 +772,6 @@ VALUES func testBatchSetScripts(t *testing.T, ds *Datastore) { ctx := context.Background() - now := time.Now().UTC().Truncate(time.Second) - insertResults := func(t *testing.T, hostID uint, scriptID uint, createdAt time.Time, execID string, exitCode *int64) { - stmt := ` -INSERT INTO - host_script_results (%s host_id, created_at, execution_id, exit_code, output) -VALUES - (%s ?,?,?,?,?)` - - args := []interface{}{} - if scriptID == 0 { - stmt = fmt.Sprintf(stmt, "", "") - } else { - stmt = fmt.Sprintf(stmt, "script_id,", "?,") - args = append(args, scriptID) - } - args = append(args, hostID, createdAt, execID, exitCode, "") - - ExecAdhocSQL(t, ds, func(tx sqlx.ExtContext) error { - _, err := tx.ExecContext(ctx, stmt, args...) - return err - }) - } - applyAndExpect := func(newSet []*fleet.Script, tmID *uint, want []*fleet.Script) map[string]uint { responseFromSet, err := ds.BatchSetScripts(ctx, tmID, newSet) require.NoError(t, err) @@ -873,8 +895,17 @@ VALUES require.Equal(t, n1WithTeamID, *teamPolicy.ScriptID) // add pending scripts on team and no-team and confirm they're shown as pending - insertResults(t, 44, n1WithTeamID, now.Add(-2*time.Minute), "execution-n1t1-1", nil) - insertResults(t, 45, n1WithNoTeamId, now.Add(-2*time.Minute), "execution-n1nt1-1", nil) + _, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ + HostID: 44, + ScriptID: &n1WithTeamID, + }) + require.NoError(t, err) + _, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ + HostID: 45, + ScriptID: &n1WithNoTeamId, + }) + require.NoError(t, err) + pending, err := ds.ListPendingHostScriptExecutions(ctx, 44, false) require.NoError(t, err) require.Len(t, pending, 1) @@ -1557,32 +1588,35 @@ func testDeletePendingHostScriptExecutionsForPolicy(t *testing.T, ds *Datastore) require.NoError(t, err) require.Equal(t, 1, len(pending)) - // test not pending host script execution for correct policy - scriptExecution, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ - HostID: 1, - ScriptContents: "echo", - UserID: &user.ID, - PolicyID: &p1.ID, - SyncRequest: true, - ScriptID: &script1.ID, - }) - require.NoError(t, err) - ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - _, err = q.ExecContext(ctx, `UPDATE host_script_results SET exit_code = 1 WHERE id = ?`, scriptExecution.ID) + // TODO(mna): adjust test once script execution via unified queue is implemented + /* + // test not pending host script execution for correct policy + scriptExecution, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{ + HostID: 1, + ScriptContents: "echo", + UserID: &user.ID, + PolicyID: &p1.ID, + SyncRequest: true, + ScriptID: &script1.ID, + }) require.NoError(t, err) - return nil - }) + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err = q.ExecContext(ctx, `UPDATE host_script_results SET exit_code = 1 WHERE id = ?`, scriptExecution.ID) + require.NoError(t, err) + return nil + }) - err = ds.deletePendingHostScriptExecutionsForPolicy(ctx, &team1.ID, p1.ID) - require.NoError(t, err) + err = ds.deletePendingHostScriptExecutionsForPolicy(ctx, &team1.ID, p1.ID) + require.NoError(t, err) - var count int - err = sqlx.GetContext( - ctx, - ds.reader(ctx), - &count, - "SELECT count(1) FROM host_script_results WHERE id = ?", - scriptExecution.ID, - ) - require.Equal(t, 1, count) + var count int + err = sqlx.GetContext( + ctx, + ds.reader(ctx), + &count, + "SELECT count(1) FROM host_script_results WHERE id = ?", + scriptExecution.ID, + ) + require.Equal(t, 1, count) + */ } diff --git a/server/datastore/mysql/setup_experience.go b/server/datastore/mysql/setup_experience.go index a4ff177352b9..92425ba76182 100644 --- a/server/datastore/mysql/setup_experience.go +++ b/server/datastore/mysql/setup_experience.go @@ -305,6 +305,9 @@ func questionMarks(number int) string { } func (ds *Datastore) ListSetupExperienceResultsByHostUUID(ctx context.Context, hostUUID string) ([]*fleet.SetupExperienceStatusResult, error) { + // TODO(mna): this references the host software installs execution id, see if/how it + // impacts the upcoming queue (might be no impact if there's no FK, as the execution + // id is constant in the upcoming -> exec flow). const stmt = ` SELECT sesr.id, @@ -337,6 +340,7 @@ WHERE host_uuid = ? } func (ds *Datastore) UpdateSetupExperienceStatusResult(ctx context.Context, status *fleet.SetupExperienceStatusResult) error { + // TODO(mna): consider if this impacts upcoming queue const stmt = ` UPDATE setup_experience_status_results SET @@ -564,6 +568,7 @@ func (ds *Datastore) MaybeUpdateSetupExperienceVPPStatus(ctx context.Context, ho } func (ds *Datastore) MaybeUpdateSetupExperienceSoftwareInstallStatus(ctx context.Context, hostUUID string, executionID string, status fleet.SetupExperienceStatusResultStatus) (bool, error) { + // TODO(mna): consider if this impacts upcoming queue selectStmt := "SELECT id FROM setup_experience_status_results WHERE host_uuid = ? AND host_software_installs_execution_id = ?" updateStmt := "UPDATE setup_experience_status_results SET status = ? WHERE id = ?" diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 6636ac24bccb..33c3cd010d53 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -2270,13 +2270,20 @@ INNER JOIN software_cve scve ON scve.software_id = s.id hs.host_id = :host_id AND s.title_id = st.id ) OR `, onlyVulnerableJoin) + + // TODO(uniq): refactor vppHostStatusNamedQuery to use the same logic as GetSummaryHostVPPAppInstalls? status := fmt.Sprintf(`COALESCE(%s, %s)`, "hsi.last_status", vppAppHostStatusNamedQuery("hvsi", "ncr", "")) + if opts.OnlyAvailableForInstall { // Get software that has a package/VPP installer but was not installed with Fleet softwareIsInstalledOnHostClause = fmt.Sprintf(` %s IS NULL AND (si.id IS NOT NULL OR vat.adam_id IS NOT NULL) AND %s`, status, softwareIsInstalledOnHostClause) } + // TODO(uniq): this query is super complex, not even sure where upcoming activities fit in, but I think it does. + // Looks like it might impact upcoming software installs, scripts and VPP apps. May need to review the whole query + // to take a different approach, this is becoming unmaintainable. + // this statement lists only the software that is reported as installed on // the host or has been attempted to be installed on the host. stmtInstalled := fmt.Sprintf(` @@ -2440,6 +2447,7 @@ INNER JOIN software_cve scve ON scve.software_id = s.id // attempted to be installed on the host, but that is available to be // installed on the host's platform. + // TODO(uniq): I think this should exclude software and VPP apps that is pending in upcoming activities stmtAvailable := fmt.Sprintf(` SELECT st.id, @@ -2883,25 +2891,34 @@ func (ds *Datastore) SetHostSoftwareInstallResult(ctx context.Context, result *f return output } - res, err := ds.writer(ctx).ExecContext(ctx, stmt, - truncateOutput(result.PreInstallConditionOutput), - result.InstallScriptExitCode, - truncateOutput(result.InstallScriptOutput), - result.PostInstallScriptExitCode, - truncateOutput(result.PostInstallScriptOutput), - result.InstallUUID, - result.HostID, - ) - if err != nil { - return ctxerr.Wrap(ctx, err, "update host software installation result") - } - if n, _ := res.RowsAffected(); n == 0 { - return ctxerr.Wrap(ctx, notFound("HostSoftwareInstall").WithName(result.InstallUUID), "host software installation not found") - } - return nil + err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + res, err := tx.ExecContext(ctx, stmt, + truncateOutput(result.PreInstallConditionOutput), + result.InstallScriptExitCode, + truncateOutput(result.InstallScriptOutput), + result.PostInstallScriptExitCode, + truncateOutput(result.PostInstallScriptOutput), + result.InstallUUID, + result.HostID, + ) + if err != nil { + return ctxerr.Wrap(ctx, err, "update host software installation result") + } + if n, _ := res.RowsAffected(); n == 0 { + return ctxerr.Wrap(ctx, notFound("HostSoftwareInstall").WithName(result.InstallUUID), "host software installation not found") + } + + if _, err := ds.activateNextUpcomingActivity(ctx, tx, result.HostID, result.InstallUUID); err != nil { + return ctxerr.Wrap(ctx, err, "activate next activity") + } + return nil + }) + return err } func getInstalledByFleetSoftwareTitles(ctx context.Context, qc sqlx.QueryerContext, hostID uint) ([]fleet.SoftwareTitle, error) { + // TODO(uniq): this only returns installed software, so no impact on upcoming queue + // We are overloading vpp_apps_count to indicate whether installed title is a VPP app or not. const stmt = ` SELECT @@ -2948,6 +2965,7 @@ WHERE hvsi.removed = 0 AND ncr.status = :mdm_status_acknowledged } func markHostSoftwareInstallsRemoved(ctx context.Context, ex sqlx.ExtContext, hostID uint, titleIDs []uint) error { + // TODO(uniq): I think this only matters for non-pending installs, so no impact on upcoming queue const stmt = ` UPDATE host_software_installs hsi INNER JOIN software_installers si ON hsi.software_installer_id = si.id @@ -2966,6 +2984,7 @@ WHERE hsi.host_id = ? AND st.id IN (?) } func markHostVPPSoftwareInstallsRemoved(ctx context.Context, ex sqlx.ExtContext, hostID uint, titleIDs []uint) error { + // TODO(uniq): I think this only matters for non-pending installs, so no impact on upcoming queue const stmt = ` UPDATE host_vpp_software_installs hvsi INNER JOIN vpp_apps vap ON hvsi.adam_id = vap.adam_id AND hvsi.platform = vap.platform diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 2ad8aa37f5cb..1042edb9a8bc 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -12,28 +12,44 @@ import ( "github.com/fleetdm/fleet/v4/server/authz" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/go-kit/kit/log/level" + "github.com/go-kit/log/level" "github.com/google/uuid" "github.com/jmoiron/sqlx" ) func (ds *Datastore) ListPendingSoftwareInstalls(ctx context.Context, hostID uint) ([]string, error) { - const stmt = ` - SELECT - execution_id - FROM - host_software_installs - WHERE - host_id = ? - AND - host_deleted_at IS NULL - AND - status = ? - ORDER BY - created_at ASC -` + return ds.listUpcomingSoftwareInstalls(ctx, hostID, false) +} + +func (ds *Datastore) ListReadyToExecuteSoftwareInstalls(ctx context.Context, hostID uint) ([]string, error) { + return ds.listUpcomingSoftwareInstalls(ctx, hostID, true) +} + +func (ds *Datastore) listUpcomingSoftwareInstalls(ctx context.Context, hostID uint, onlyReadyToExecute bool) ([]string, error) { + extraWhere := "" + if onlyReadyToExecute { + extraWhere = " AND activated_at IS NOT NULL" + } + + stmt := fmt.Sprintf(` + SELECT + execution_id + FROM ( + SELECT + execution_id, + IF(activated_at IS NULL, 0, 1) as topmost, + priority, + created_at + FROM + upcoming_activities + WHERE + host_id = ? AND + activity_type = 'software_install' + %s + ORDER BY topmost DESC, priority ASC, created_at ASC) as t +`, extraWhere) var results []string - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, stmt, hostID, fleet.SoftwareInstallPending); err != nil { + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, stmt, hostID); err != nil { return nil, ctxerr.Wrap(ctx, err, "list pending software installs") } return results, nil @@ -65,10 +81,43 @@ func (ds *Datastore) GetSoftwareInstallDetails(ctx context.Context, executionId script_contents pisnt ON pisnt.id = si.post_install_script_content_id WHERE - hsi.execution_id = ?` + hsi.execution_id = ? + + UNION + + SELECT + ua.host_id AS host_id, + ua.execution_id AS execution_id, + siua.software_installer_id AS installer_id, + ua.payload->'$.self_service' AS self_service, + COALESCE(si.pre_install_query, '') AS pre_install_condition, + inst.contents AS install_script, + uninst.contents AS uninstall_script, + COALESCE(pisnt.contents, '') AS post_install_script + FROM + upcoming_activities ua + INNER JOIN + software_install_upcoming_activities siua + ON ua.id = siua.upcoming_activity_id + INNER JOIN + software_installers si + ON siua.software_installer_id = si.id + LEFT OUTER JOIN + script_contents inst + ON inst.id = si.install_script_content_id + LEFT OUTER JOIN + script_contents uninst + ON uninst.id = si.uninstall_script_content_id + LEFT OUTER JOIN + script_contents pisnt + ON pisnt.id = si.post_install_script_content_id + WHERE + ua.execution_id = ? AND + ua.activated_at IS NULL -- if already activated, then it is covered by the other SELECT +` result := &fleet.SoftwareInstallDetails{} - if err := sqlx.GetContext(ctx, ds.reader(ctx), result, stmt, executionId); err != nil { + if err := sqlx.GetContext(ctx, ds.reader(ctx), result, stmt, executionId, executionId); err != nil { if err == sql.ErrNoRows { return nil, ctxerr.Wrap(ctx, notFound("SoftwareInstallerDetails").WithName(executionId), "get software installer details") } @@ -458,6 +507,9 @@ func (ds *Datastore) SaveInstallerUpdates(ctx context.Context, payload *fleet.Up } func (ds *Datastore) ValidateOrbitSoftwareInstallerAccess(ctx context.Context, hostID uint, installerID uint) (bool, error) { + // NOTE: this is ok to only look in host_software_installs (and ignore + // upcoming_activities), because orbit should not be able to get the + // installer until it is ready to install. query := ` SELECT 1 FROM @@ -666,13 +718,18 @@ func (ds *Datastore) DeleteSoftwareInstaller(ctx context.Context, id uint) error }) } -// deletePendingSoftwareInstallsForPolicy should be called after a policy is deleted to remove any pending software installs +// deletePendingSoftwareInstallsForPolicy should be called after a policy is +// deleted to remove any pending software installs func (ds *Datastore) deletePendingSoftwareInstallsForPolicy(ctx context.Context, teamID *uint, policyID uint) error { var globalOrTeamID uint if teamID != nil { globalOrTeamID = *teamID } + // NOTE(mna): I'm adding the deletion for the upcoming_activities too, but I + // don't think the existing code works as intended anyway as the + // host_software_installs.policy_id column has a ON DELETE SET NULL foreign + // key, so the deletion statement will not find any row. const deleteStmt = ` DELETE FROM host_software_installs @@ -688,18 +745,59 @@ func (ds *Datastore) deletePendingSoftwareInstallsForPolicy(ctx context.Context, return ctxerr.Wrap(ctx, err, "delete pending software installs for policy") } + const deleteUAStmt = ` + DELETE FROM + upcoming_activities + USING + upcoming_activities + INNER JOIN software_install_upcoming_activities siua + ON upcoming_activities.id = siua.upcoming_activity_id + WHERE + upcoming_activities.activity_type = 'software_install' AND + siua.policy_id = ? AND + siua.software_installer_id IN ( + SELECT id FROM software_installers WHERE global_or_team_id = ? + ) + ` + _, err = ds.writer(ctx).ExecContext(ctx, deleteUAStmt, policyID, globalOrTeamID) + if err != nil { + return ctxerr.Wrap(ctx, err, "delete upcoming software installs for policy") + } + return nil } -func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint, selfService bool, policyID *uint) (string, error) { +func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint, opts fleet.HostSoftwareInstallOptions) (string, error) { const ( - getInstallerStmt = `SELECT filename, "version", title_id, COALESCE(st.name, '[deleted title]') title_name - FROM software_installers si LEFT JOIN software_titles st ON si.title_id = st.id WHERE si.id = ?` - insertStmt = ` - INSERT INTO host_software_installs - (execution_id, host_id, software_installer_id, user_id, self_service, policy_id, installer_filename, version, software_title_id, software_title_name) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ` + getInstallerStmt = ` +SELECT + filename, "version", title_id, COALESCE(st.name, '[deleted title]') title_name +FROM + software_installers si + LEFT JOIN software_titles st + ON si.title_id = st.id +WHERE si.id = ?` + + insertUAStmt = ` +INSERT INTO upcoming_activities + (host_id, priority, user_id, fleet_initiated, activity_type, execution_id, payload) +VALUES + (?, ?, ?, ?, 'software_install', ?, + JSON_OBJECT( + 'self_service', ?, + 'installer_filename', ?, + 'version', ?, + 'software_title_name', ?, + 'user', (SELECT JSON_OBJECT('name', name, 'email', email, 'gravatar_url', gravatar_url) FROM users WHERE id = ?) + ) + )` + + insertSIUAStmt = ` +INSERT INTO software_install_upcoming_activities + (upcoming_activity_id, software_installer_id, policy_id, software_title_id) +VALUES + (?, ?, ?, ?)` + hostExistsStmt = `SELECT 1 FROM hosts WHERE id = ?` ) @@ -729,24 +827,45 @@ func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID ui } var userID *uint - if ctxUser := authz.UserFromContext(ctx); ctxUser != nil { + if ctxUser := authz.UserFromContext(ctx); ctxUser != nil && opts.PolicyID == nil { userID = &ctxUser.ID } - installID := uuid.NewString() - _, err = ds.writer(ctx).ExecContext(ctx, insertStmt, - installID, - hostID, - softwareInstallerID, - userID, - selfService, - policyID, - installerDetails.Filename, - installerDetails.Version, - installerDetails.TitleID, - installerDetails.TitleName, - ) + execID := uuid.NewString() + + err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + res, err := tx.ExecContext(ctx, insertUAStmt, + hostID, + opts.Priority(), + userID, + opts.IsFleetInitiated(), + execID, + opts.SelfService, + installerDetails.Filename, + installerDetails.Version, + installerDetails.TitleName, + userID, + ) + if err != nil { + return ctxerr.Wrap(ctx, err, "insert software install request") + } + + activityID, _ := res.LastInsertId() + _, err = tx.ExecContext(ctx, insertSIUAStmt, + activityID, + softwareInstallerID, + opts.PolicyID, + installerDetails.TitleID, + ) + if err != nil { + return ctxerr.Wrap(ctx, err, "insert software install request join table") + } - return installID, ctxerr.Wrap(ctx, err, "inserting new install software request") + if _, err := ds.activateNextUpcomingActivity(ctx, tx, hostID, ""); err != nil { + return ctxerr.Wrap(ctx, err, "activate next activity") + } + return nil + }) + return execID, ctxerr.Wrap(ctx, err, "inserting new install software request") } func (ds *Datastore) ProcessInstallerUpdateSideEffects(ctx context.Context, installerID uint, wasMetadataUpdated bool, wasPackageUpdated bool) error { @@ -770,6 +889,16 @@ func (ds *Datastore) runInstallerUpdateSideEffectsInTransaction(ctx context.Cont if err != nil { return ctxerr.Wrap(ctx, err, "delete pending host software installs/uninstalls") } + + _, err = tx.ExecContext(ctx, `DELETE FROM upcoming_activities + USING + upcoming_activities + INNER JOIN software_install_upcoming_activities siua + ON upcoming_activities.id = siua.upcoming_activity_id + WHERE siua.software_installer_id = ? AND activity_type IN ('software_install', 'software_uninstall')`, installerID) + if err != nil { + return ctxerr.Wrap(ctx, err, "delete upcoming host software installs/uninstalls") + } } if wasPackageUpdated { // hide existing install counts @@ -787,11 +916,26 @@ func (ds *Datastore) InsertSoftwareUninstallRequest(ctx context.Context, executi const ( getInstallerStmt = `SELECT title_id, COALESCE(st.name, '[deleted title]') title_name FROM software_installers si LEFT JOIN software_titles st ON si.title_id = st.id WHERE si.id = ?` - insertStmt = ` - INSERT INTO host_software_installs - (execution_id, host_id, software_installer_id, user_id, uninstall, installer_filename, software_title_id, software_title_name, version) - VALUES (?, ?, ?, ?, 1, '', ?, ?, 'unknown') - ` + + insertUAStmt = ` +INSERT INTO upcoming_activities + (host_id, priority, user_id, fleet_initiated, activity_type, execution_id, payload) +VALUES + (?, ?, ?, ?, 'software_uninstall', ?, + JSON_OBJECT( + 'installer_filename', '', + 'version', 'unknown', + 'software_title_name', ?, + 'user', (SELECT JSON_OBJECT('name', name, 'email', email, 'gravatar_url', gravatar_url) FROM users WHERE id = ?) + ) + )` + + insertSIUAStmt = ` +INSERT INTO software_install_upcoming_activities + (upcoming_activity_id, software_installer_id, software_title_id) +VALUES + (?, ?, ?)` + hostExistsStmt = `SELECT 1 FROM hosts WHERE id = ?` ) @@ -818,17 +962,41 @@ func (ds *Datastore) InsertSoftwareUninstallRequest(ctx context.Context, executi } var userID *uint + fleetInitiated := true if ctxUser := authz.UserFromContext(ctx); ctxUser != nil { userID = &ctxUser.ID + fleetInitiated = false } - _, err = ds.writer(ctx).ExecContext(ctx, insertStmt, - executionID, - hostID, - softwareInstallerID, - userID, - installerDetails.TitleID, - installerDetails.TitleName, - ) + + err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + res, err := tx.ExecContext(ctx, insertUAStmt, + hostID, + 0, // Uninstalls are never used in setup experience, so always default priority + userID, + fleetInitiated, + executionID, + installerDetails.TitleName, + userID, + ) + if err != nil { + return err + } + + activityID, _ := res.LastInsertId() + _, err = tx.ExecContext(ctx, insertSIUAStmt, + activityID, + softwareInstallerID, + installerDetails.TitleID, + ) + if err != nil { + return err + } + + if _, err := ds.activateNextUpcomingActivity(ctx, tx, hostID, ""); err != nil { + return ctxerr.Wrap(ctx, err, "activate next activity") + } + return nil + }) return ctxerr.Wrap(ctx, err, "inserting new uninstall software request") } @@ -857,8 +1025,40 @@ FROM host_software_installs hsi LEFT JOIN software_titles st ON hsi.software_title_id = st.id WHERE - hsi.execution_id = :execution_id - ` + hsi.execution_id = :execution_id AND + hsi.uninstall = 0 + +UNION + +SELECT + ua.execution_id AS execution_id, + NULL AS pre_install_query_output, + NULL AS post_install_script_output, + NULL AS install_script_output, + ua.host_id AS host_id, + COALESCE(st.name, ua.payload->>'$.software_title_name') AS software_title, + siua.software_title_id, + 'pending_install' AS status, + ua.payload->>'$.installer_filename' AS software_package, + ua.user_id AS user_id, + NULL AS post_install_script_exit_code, + NULL AS install_script_exit_code, + ua.payload->'$.self_service' AS self_service, + NULL AS host_deleted_at, + siua.policy_id AS policy_id, + ua.created_at as created_at, + ua.updated_at as updated_at +FROM + upcoming_activities ua + INNER JOIN software_install_upcoming_activities siua + ON ua.id = siua.upcoming_activity_id + LEFT JOIN software_titles st + ON siua.software_title_id = st.id +WHERE + ua.execution_id = :execution_id AND + ua.activity_type = 'software_install' AND + ua.activated_at IS NULL -- if already activated, covered by the other SELECT +` stmt, args, err := sqlx.Named(query, map[string]any{ "execution_id": resultsUUID, @@ -882,7 +1082,48 @@ WHERE func (ds *Datastore) GetSummaryHostSoftwareInstalls(ctx context.Context, installerID uint) (*fleet.SoftwareInstallerStatusSummary, error) { var dest fleet.SoftwareInstallerStatusSummary - stmt := ` + // TODO(uniq): AFAICT we don't have uniqueness for host_id + title_id in upcoming or + // past activities. In the past the max(id) approach was "good enough" as a proxy for the most + // recent activity since we didn't really need to worry about the order of activities. + // Moving to a time-based approach would be more accurate but would require a more complex and + // potentially slower query. + + stmt := `WITH + +-- select most recent upcoming activities for each host +upcoming AS ( + SELECT + max(ua.id) AS upcoming_id, -- ensure we use only the most recent attempt for each host + host_id + FROM + upcoming_activities ua + JOIN software_install_upcoming_activities siua ON ua.id = siua.upcoming_activity_id + JOIN hosts h ON host_id = h.id + WHERE + activity_type IN('software_install', 'software_uninstall') + AND software_installer_id = :installer_id + GROUP BY + host_id +), + +-- select most recent past activities for each host +past AS ( + SELECT + max(hsi.id) AS past_id, -- ensure we use only the most recent attempt for each host + host_id + FROM + host_software_installs hsi + JOIN hosts h ON host_id = h.id + WHERE + software_installer_id = :installer_id + AND host_id NOT IN(SELECT host_id FROM upcoming) -- antijoin to exclude hosts with upcoming activities + AND host_deleted_at IS NULL + AND removed = 0 + GROUP BY + host_id +) + +-- count each status SELECT COALESCE(SUM( IF(status = :software_status_pending_install, 1, 0)), 0) AS pending_install, COALESCE(SUM( IF(status = :software_status_failed_install, 1, 0)), 0) AS failed_install, @@ -890,23 +1131,22 @@ SELECT COALESCE(SUM( IF(status = :software_status_failed_uninstall, 1, 0)), 0) AS failed_uninstall, COALESCE(SUM( IF(status = :software_status_installed, 1, 0)), 0) AS installed FROM ( + +-- union most recent past and upcoming activities after joining to get statuses for most recent activities SELECT - software_installer_id, + past.host_id, status FROM - host_software_installs hsi -WHERE - software_installer_id = :installer_id - AND id IN( - SELECT - max(id) -- ensure we use only the most recently created install attempt for each host - FROM host_software_installs - WHERE - software_installer_id = :installer_id - AND host_deleted_at IS NULL - AND removed = 0 - GROUP BY - host_id)) s` + past + JOIN host_software_installs hsi ON hsi.id = past_id +UNION +SELECT + upcoming.host_id, + IF(activity_type = 'software_install', :software_status_pending_install, :software_status_pending_uninstall) AS status + FROM + upcoming + JOIN software_install_upcoming_activities siua ON upcoming_id = siua.upcoming_activity_id + JOIN upcoming_activities ua ON ua.id = upcoming_id) t` query, args, err := sqlx.Named(stmt, map[string]interface{}{ "installer_id": installerID, @@ -929,15 +1169,43 @@ WHERE } func (ds *Datastore) vppAppJoin(appID fleet.VPPAppID, status fleet.SoftwareInstallerStatus) (string, []interface{}, error) { - // Since VPP does not have uninstaller yet, we map the generic pending/failed statuses to the install statuses - switch status { - case fleet.SoftwarePending: - status = fleet.SoftwareInstallPending - case fleet.SoftwareFailed: - status = fleet.SoftwareInstallFailed - default: - // no change + // for pending status, we'll join through upcoming_activities + if status == fleet.SoftwarePending || status == fleet.SoftwareInstallPending || status == fleet.SoftwareUninstallPending { + stmt := `JOIN ( +SELECT DISTINCT + host_id +FROM + upcoming_activities ua + JOIN vpp_app_upcoming_activities vppua ON ua.id = vppua.upcoming_activity_id +WHERE + %s) hss ON hss.host_id = h.id` + + filter := "vppua.adam_id = ? AND vppua.platform = ?" + switch status { + case fleet.SoftwareInstallPending: + filter += " AND ua.activity_type = 'vpp_app_install'" + case fleet.SoftwareUninstallPending: + // TODO: Update this when VPP supports uninstall, for now we map uninstall to install to preserve existing behavior of VPP filters + filter += " AND ua.activity_type = 'vpp_app_install'" + default: + // no change, we're just filtering by app_id and platform so it will pick up any + // activity type that is associated with the app (i.e. both install and uninstall) + } + + return fmt.Sprintf(stmt, filter), []interface{}{appID.AdamID, appID.Platform}, nil + } + + // TODO: Update this when VPP supports uninstall so that we map for now we map the generic failed status to the install statuses + if status == fleet.SoftwareFailed { + status = fleet.SoftwareInstallFailed // TODO: When VPP supports uninstall this should become STATUS IN ('failed_install', 'failed_uninstall') } + + // TODO(uniq): AFAICT we don't have uniqueness for host_id + title_id in upcoming or + // past activities. In the past the max(id) approach was "good enough" as a proxy for the most + // recent activity since we didn't really need to worry about the order of activities. + // Moving to a time-based approach would be more accurate but would require a more complex and + // potentially slower query. + stmt := fmt.Sprintf(`JOIN ( SELECT host_id @@ -956,7 +1224,7 @@ WHERE GROUP BY host_id, adam_id) AND (%s) = :status) hss ON hss.host_id = h.id -`, vppAppHostStatusNamedQuery("hvsi", "ncr", "")) +`, vppAppHostStatusNamedQuery("hvsi", "ncr", "")) // TODO(uniq): refactor vppHostStatusNamedQuery to use the same logic as GetSummaryHostVPPAppInstalls? return sqlx.Named(stmt, map[string]interface{}{ "status": status, @@ -971,77 +1239,134 @@ WHERE }) } -func (ds *Datastore) softwareInstallerJoin(installerID uint, status fleet.SoftwareInstallerStatus) (string, []interface{}, error) { +func (ds *Datastore) softwareInstallerJoin(titleID uint, status fleet.SoftwareInstallerStatus) (string, []interface{}, error) { + // for pending status, we'll join through upcoming_activities + if status == fleet.SoftwarePending || status == fleet.SoftwareInstallPending || status == fleet.SoftwareUninstallPending { + stmt := `JOIN ( +SELECT DISTINCT + host_id +FROM + upcoming_activities ua + JOIN software_install_upcoming_activities siua ON ua.id = siua.upcoming_activity_id +WHERE + %s) hss ON hss.host_id = h.id` + + filter := "siua.software_title_id = ?" + switch status { + case fleet.SoftwareInstallPending: + filter += " AND ua.activity_type = 'software_install'" + case fleet.SoftwareUninstallPending: + filter += " AND ua.activity_type = 'software_uninstall'" + default: + // no change + } + + return fmt.Sprintf(stmt, filter), []interface{}{titleID}, nil + } + + // for non-pending statuses, we'll join through host_software_installs filtered by the status statusFilter := "hsi.status = :status" - var status2 fleet.SoftwareInstallerStatus - switch status { - case fleet.SoftwarePending: - status = fleet.SoftwareInstallPending - status2 = fleet.SoftwareUninstallPending - case fleet.SoftwareFailed: - status = fleet.SoftwareInstallFailed - status2 = fleet.SoftwareUninstallFailed - default: - // no change - } - if status2 != "" { - statusFilter = "hsi.status IN (:status, :status2)" + if status == fleet.SoftwareFailed { + // failed is a special case, we must include both install and uninstall failures + statusFilter = "hsi.status IN (:installFailed, :uninstallFailed)" } + + // TODO(uniq): AFAICT we don't have uniqueness for host_id + title_id in upcoming or + // past activities. In the past the max(id) approach was "good enough" as a proxy for the most + // recent activity since we didn't really need to worry about the order of activities. + // Moving to a time-based approach would be more accurate but would require a more complex and + // potentially slower query. + stmt := fmt.Sprintf(`JOIN ( SELECT host_id FROM host_software_installs hsi WHERE - software_installer_id = :installer_id + software_title_id = :title_id AND hsi.id IN( SELECT max(id) -- ensure we use only the most recent install attempt for each host FROM host_software_installs WHERE - software_installer_id = :installer_id + host_id = hsi.host_id + AND software_title_id = :title_id AND removed = 0 GROUP BY - host_id, software_installer_id) + host_id, software_title_id) AND %s) hss ON hss.host_id = h.id `, statusFilter) return sqlx.Named(stmt, map[string]interface{}{ - "status": status, - "status2": status2, - "installer_id": installerID, + "status": status, + "installFailed": fleet.SoftwareInstallFailed, + "uninstallFailed": fleet.SoftwareUninstallFailed, + // TODO(uniq): prior code was joining based on installer id but based on how list options are parsed [1] it seems like this should be the title id + // [1] https://github.com/fleetdm/fleet/blob/8aecae4d853829cb6e7f828099a4f0953643cf18/server/datastore/mysql/hosts.go#L1088-L1089 + "title_id": titleID, }) } func (ds *Datastore) GetHostLastInstallData(ctx context.Context, hostID, installerID uint) (*fleet.HostLastInstallData, error) { + hostLastInstall, err := ds.getLatestUpcomingInstall(ctx, hostID, installerID) + if err != nil && errors.Is(err, sql.ErrNoRows) { + hostLastInstall, err = ds.getLatestPastInstall(ctx, hostID, installerID) + } + + if err != nil && errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + + return hostLastInstall, err +} + +func (ds *Datastore) getLatestUpcomingInstall(ctx context.Context, hostID, installerID uint) (*fleet.HostLastInstallData, error) { + var hostLastInstall fleet.HostLastInstallData stmt := ` - SELECT execution_id, hsi.status - FROM host_software_installs hsi - WHERE hsi.id = ( - SELECT - MAX(id) - FROM host_software_installs - WHERE - software_installer_id = :installer_id AND host_id = :host_id AND host_deleted_at IS NULL - GROUP BY - host_id, software_installer_id) -` +SELECT + execution_id, + 'pending_install' AS status +FROM + upcoming_activities +WHERE + id = ( + SELECT + MAX(ua.id) + FROM + upcoming_activities ua + JOIN + software_install_upcoming_activities siua ON ua.id = siua.upcoming_activity_id + WHERE + ua.activity_type = 'software_install' AND ua.host_id = ? AND siua.software_installer_id = ?)` - stmt, args, err := sqlx.Named(stmt, map[string]interface{}{ - "host_id": hostID, - "installer_id": installerID, - }) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "build named query to get host last install data") + if err := sqlx.GetContext(ctx, ds.reader(ctx), &hostLastInstall, stmt, hostID, installerID); err != nil { + return nil, ctxerr.Wrap(ctx, err, "get latest upcoming install") } + return &hostLastInstall, nil +} + +func (ds *Datastore) getLatestPastInstall(ctx context.Context, hostID, installerID uint) (*fleet.HostLastInstallData, error) { var hostLastInstall fleet.HostLastInstallData - if err := sqlx.GetContext(ctx, ds.reader(ctx), &hostLastInstall, stmt, args...); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - return nil, ctxerr.Wrap(ctx, err, "get host last install data") + stmt := ` +SELECT + execution_id, + status +FROM + host_software_installs +WHERE + id = ( + SELECT + MAX(hsi.id) + FROM + host_software_installs hsi + WHERE + hsi.host_id = ? AND hsi.software_installer_id = ?)` + + if err := sqlx.GetContext(ctx, ds.reader(ctx), &hostLastInstall, stmt, hostID, installerID); err != nil { + return nil, ctxerr.Wrap(ctx, err, "get lastest past install") } + return &hostLastInstall, nil } @@ -1093,24 +1418,36 @@ WHERE const deleteAllPendingUninstallScriptExecutions = ` DELETE FROM host_script_results WHERE execution_id IN ( SELECT execution_id FROM host_software_installs WHERE status = 'pending_uninstall' - AND software_installer_id IN ( - SELECT id FROM software_installers WHERE global_or_team_id = ? - ) + AND software_installer_id IN ( + SELECT id FROM software_installers WHERE global_or_team_id = ? + ) ) ` - const deleteAllPendingSoftwareInstalls = ` + const deleteAllPendingSoftwareInstallsHSI = ` DELETE FROM host_software_installs - WHERE status IN('pending_install', 'pending_uninstall') - AND software_installer_id IN ( - SELECT id FROM software_installers WHERE global_or_team_id = ? - ) + WHERE status IN('pending_install', 'pending_uninstall') + AND software_installer_id IN ( + SELECT id FROM software_installers WHERE global_or_team_id = ? + ) +` + + const deleteAllPendingSoftwareInstallsUA = ` + DELETE FROM upcoming_activities + USING upcoming_activities + INNER JOIN software_install_upcoming_activities siua + ON upcoming_activities.id = siua.upcoming_activity_id + WHERE + activity_type IN ('software_install', 'software_uninstall') AND + siua.software_installer_id IN ( + SELECT id FROM software_installers WHERE global_or_team_id = ? + ) ` const markAllSoftwareInstallsAsRemoved = ` UPDATE host_software_installs SET removed = TRUE - WHERE status IS NOT NULL AND host_deleted_at IS NULL - AND software_installer_id IN ( - SELECT id FROM software_installers WHERE global_or_team_id = ? - ) + WHERE status IS NOT NULL AND host_deleted_at IS NULL + AND software_installer_id IN ( + SELECT id FROM software_installers WHERE global_or_team_id = ? + ) ` const deleteAllInstallersInTeam = ` @@ -1123,17 +1460,28 @@ WHERE const deletePendingUninstallScriptExecutionsNotInList = ` DELETE FROM host_script_results WHERE execution_id IN ( SELECT execution_id FROM host_software_installs WHERE status = 'pending_uninstall' - AND software_installer_id IN ( - SELECT id FROM software_installers WHERE global_or_team_id = ? AND title_id NOT IN (?) - ) + AND software_installer_id IN ( + SELECT id FROM software_installers WHERE global_or_team_id = ? AND title_id NOT IN (?) + ) ) ` - const deletePendingSoftwareInstallsNotInList = ` + const deletePendingSoftwareInstallsNotInListHSI = ` DELETE FROM host_software_installs - WHERE status IN('pending_install', 'pending_uninstall') - AND software_installer_id IN ( - SELECT id FROM software_installers WHERE global_or_team_id = ? AND title_id NOT IN (?) - ) + WHERE status IN('pending_install', 'pending_uninstall') + AND software_installer_id IN ( + SELECT id FROM software_installers WHERE global_or_team_id = ? AND title_id NOT IN (?) + ) +` + const deletePendingSoftwareInstallsNotInListUA = ` + DELETE FROM upcoming_activities + USING upcoming_activities + INNER JOIN software_install_upcoming_activities siua + ON upcoming_activities.id = siua.upcoming_activity_id + WHERE + activity_type IN ('software_install', 'software_uninstall') AND + siua.software_installer_id IN ( + SELECT id FROM software_installers WHERE global_or_team_id = ? AND title_id NOT IN (?) + ) ` const markSoftwareInstallsNotInListAsRemoved = ` UPDATE host_software_installs SET removed = TRUE @@ -1306,10 +1654,14 @@ WHERE return ctxerr.Wrap(ctx, err, "delete all pending uninstall script executions") } - if _, err := tx.ExecContext(ctx, deleteAllPendingSoftwareInstalls, globalOrTeamID); err != nil { + if _, err := tx.ExecContext(ctx, deleteAllPendingSoftwareInstallsHSI, globalOrTeamID); err != nil { return ctxerr.Wrap(ctx, err, "delete all pending host software install records") } + if _, err := tx.ExecContext(ctx, deleteAllPendingSoftwareInstallsUA, globalOrTeamID); err != nil { + return ctxerr.Wrap(ctx, err, "delete all upcoming pending host software install records") + } + if _, err := tx.ExecContext(ctx, markAllSoftwareInstallsAsRemoved, globalOrTeamID); err != nil { return ctxerr.Wrap(ctx, err, "mark all host software installs as removed") } @@ -1370,7 +1722,7 @@ WHERE return ctxerr.Wrap(ctx, err, "delete obsolete pending uninstall script executions") } - stmt, args, err = sqlx.In(deletePendingSoftwareInstallsNotInList, globalOrTeamID, titleIDs) + stmt, args, err = sqlx.In(deletePendingSoftwareInstallsNotInListHSI, globalOrTeamID, titleIDs) if err != nil { return ctxerr.Wrap(ctx, err, "build statement to delete pending software installs") } @@ -1378,6 +1730,14 @@ WHERE return ctxerr.Wrap(ctx, err, "delete obsolete pending host software install records") } + stmt, args, err = sqlx.In(deletePendingSoftwareInstallsNotInListUA, globalOrTeamID, titleIDs) + if err != nil { + return ctxerr.Wrap(ctx, err, "build statement to delete upcoming pending software installs") + } + if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "delete obsolete upcoming pending host software install records") + } + stmt, args, err = sqlx.In(markSoftwareInstallsNotInListAsRemoved, globalOrTeamID, titleIDs) if err != nil { return ctxerr.Wrap(ctx, err, "build statement to mark obsolete host software installs as removed") @@ -1608,14 +1968,27 @@ func (ds *Datastore) HasSelfServiceSoftwareInstallers(ctx context.Context, hostP func (ds *Datastore) GetSoftwareTitleNameFromExecutionID(ctx context.Context, executionID string) (string, error) { stmt := ` - SELECT name + SELECT st.name FROM software_titles st INNER JOIN software_installers si ON si.title_id = st.id INNER JOIN host_software_installs hsi ON hsi.software_installer_id = si.id WHERE hsi.execution_id = ? + + UNION + + SELECT st.name + FROM + software_titles st + INNER JOIN software_installers si ON si.title_id = st.id + INNER JOIN software_install_upcoming_activities siua + ON siua.software_installer_id = si.id + INNER JOIN upcoming_activities ua ON ua.id = siua.upcoming_activity_id + WHERE + ua.execution_id = ? AND + ua.activity_type IN ('software_install', 'software_uninstall') ` var name string - err := sqlx.GetContext(ctx, ds.reader(ctx), &name, stmt, executionID) + err := sqlx.GetContext(ctx, ds.reader(ctx), &name, stmt, executionID, executionID) if err != nil { return "", ctxerr.Wrap(ctx, err, "get software title name from execution ID") } diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index 38d7cd81ea38..269bfc10159e 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -3,6 +3,7 @@ package mysql import ( "bytes" "context" + "database/sql" "fmt" "os" "path/filepath" @@ -40,6 +41,7 @@ func TestSoftwareInstallers(t *testing.T) { {"GetOrGenerateSoftwareInstallerTitleID", testGetOrGenerateSoftwareInstallerTitleID}, {"BatchSetSoftwareInstallersScopedViaLabels", testBatchSetSoftwareInstallersScopedViaLabels}, {"MatchOrCreateSoftwareInstallerWithAutomaticPolicies", testMatchOrCreateSoftwareInstallerWithAutomaticPolicies}, + {"GetSoftwareTitleNameFromExecutionID", testGetSoftwareTitleNameFromExecutionID}, } for _, c := range cases { @@ -52,6 +54,7 @@ func TestSoftwareInstallers(t *testing.T) { func testListPendingSoftwareInstalls(t *testing.T, ds *Datastore) { ctx := context.Background() + t.Cleanup(func() { ds.testActivateSpecificNextActivities = nil }) host1 := test.NewHost(t, ds, "host1", "1", "host1key", "host1uuid", time.Now()) host2 := test.NewHost(t, ds, "host2", "2", "host2key", "host2uuid", time.Now()) @@ -127,16 +130,40 @@ func testListPendingSoftwareInstalls(t *testing.T, ds *Datastore) { }) require.NoError(t, err) - hostInstall1, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID1, false, nil) + // ensure that nothing gets automatically activated, we want to control + // specific activation for this test + ds.testActivateSpecificNextActivities = []string{"-"} + + hostInstall1, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID1, fleet.HostSoftwareInstallOptions{}) + require.NoError(t, err) + + time.Sleep(time.Millisecond) + hostInstall2, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID2, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) - hostInstall2, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID2, false, nil) + time.Sleep(time.Millisecond) + hostInstall3, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID1, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) - hostInstall3, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID1, false, nil) + time.Sleep(time.Millisecond) + hostInstall4, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID2, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) - hostInstall4, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID2, false, nil) + pendingHost1, err := ds.ListPendingSoftwareInstalls(ctx, host1.ID) + require.NoError(t, err) + require.Equal(t, 2, len(pendingHost1)) + require.Equal(t, hostInstall1, pendingHost1[0]) + require.Equal(t, hostInstall2, pendingHost1[1]) + + pendingHost2, err := ds.ListPendingSoftwareInstalls(ctx, host2.ID) + require.NoError(t, err) + require.Equal(t, 2, len(pendingHost2)) + require.Equal(t, hostInstall3, pendingHost2[0]) + require.Equal(t, hostInstall4, pendingHost2[1]) + + // activate and set a result for hostInstall4 (installerID2) + ds.testActivateSpecificNextActivities = []string{hostInstall4} + _, err = ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), host2.ID, "") require.NoError(t, err) err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ @@ -146,7 +173,12 @@ func testListPendingSoftwareInstalls(t *testing.T, ds *Datastore) { }) require.NoError(t, err) - hostInstall5, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID2, false, nil) + // create a new pending install request on host2 for installerID2 + hostInstall5, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID2, fleet.HostSoftwareInstallOptions{}) + require.NoError(t, err) + + ds.testActivateSpecificNextActivities = []string{hostInstall5} + _, err = ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), host2.ID, "") require.NoError(t, err) err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ @@ -166,7 +198,6 @@ func testListPendingSoftwareInstalls(t *testing.T, ds *Datastore) { require.Contains(t, installDetailsList1, hostInstall1) require.Contains(t, installDetailsList1, hostInstall2) - require.Contains(t, installDetailsList2, hostInstall3) exec1, err := ds.GetSoftwareInstallDetails(ctx, hostInstall1) @@ -181,7 +212,12 @@ func testListPendingSoftwareInstalls(t *testing.T, ds *Datastore) { require.False(t, exec1.SelfService) assert.Equal(t, "goodbye MONSTER", exec1.UninstallScript) - hostInstall6, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID3, true, nil) + // add a self-service request for installerID3 on host1 + hostInstall6, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID3, fleet.HostSoftwareInstallOptions{SelfService: true}) + require.NoError(t, err) + + ds.testActivateSpecificNextActivities = []string{hostInstall6} + _, err = ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), host1.ID, "") require.NoError(t, err) err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ @@ -204,7 +240,7 @@ func testListPendingSoftwareInstalls(t *testing.T, ds *Datastore) { // Create install request, don't fulfil it, delete and restore host. // Should not appear in list of pending installs for that host. - _, err = ds.InsertSoftwareInstallRequest(ctx, host3.ID, installerID1, false, nil) + _, err = ds.InsertSoftwareInstallRequest(ctx, host3.ID, installerID1, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) err = ds.DeleteHost(ctx, host3.ID) @@ -262,7 +298,7 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { require.Equal(t, "foo.pkg", si.Name) // non-existent host - _, err = ds.InsertSoftwareInstallRequest(ctx, 12, si.InstallerID, false, nil) + _, err = ds.InsertSoftwareInstallRequest(ctx, 12, si.InstallerID, fleet.HostSoftwareInstallOptions{}) require.ErrorAs(t, err, &nfe) // Host with software install pending @@ -276,7 +312,7 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { TeamID: teamID, }) require.NoError(t, err) - _, err = ds.InsertSoftwareInstallRequest(ctx, hostPendingInstall.ID, si.InstallerID, false, nil) + _, err = ds.InsertSoftwareInstallRequest(ctx, hostPendingInstall.ID, si.InstallerID, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) // Host with software install failed @@ -290,15 +326,14 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { TeamID: teamID, }) require.NoError(t, err) - _, err = ds.InsertSoftwareInstallRequest(ctx, hostFailedInstall.ID, si.InstallerID, false, nil) + execID, err := ds.InsertSoftwareInstallRequest(ctx, hostFailedInstall.ID, si.InstallerID, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) - ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - _, err = q.ExecContext(ctx, ` - UPDATE host_software_installs SET install_script_exit_code = 1 WHERE host_id = ? AND software_installer_id = ?`, - hostFailedInstall.ID, si.InstallerID) - require.NoError(t, err) - return nil + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: hostFailedInstall.ID, + InstallUUID: execID, + InstallScriptExitCode: ptr.Int(1), }) + require.NoError(t, err) // Host with software install successful tag = "-installed" @@ -311,15 +346,14 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { TeamID: teamID, }) require.NoError(t, err) - _, err = ds.InsertSoftwareInstallRequest(ctx, hostInstalled.ID, si.InstallerID, false, nil) + execID, err = ds.InsertSoftwareInstallRequest(ctx, hostInstalled.ID, si.InstallerID, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) - ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - _, err = q.ExecContext(ctx, ` - UPDATE host_software_installs SET install_script_exit_code = 0 WHERE host_id = ? AND software_installer_id = ?`, - hostInstalled.ID, si.InstallerID) - require.NoError(t, err) - return nil + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: hostInstalled.ID, + InstallUUID: execID, + InstallScriptExitCode: ptr.Int(0), }) + require.NoError(t, err) // Host with pending uninstall tag = "-pending_uninstall" @@ -346,15 +380,15 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { TeamID: teamID, }) require.NoError(t, err) - err = ds.InsertSoftwareUninstallRequest(ctx, "uuid"+tag+tc, hostFailedUninstall.ID, si.InstallerID) + execID = "uuid" + tag + tc + err = ds.InsertSoftwareUninstallRequest(ctx, execID, hostFailedUninstall.ID, si.InstallerID) require.NoError(t, err) - ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - _, err = q.ExecContext(ctx, ` - UPDATE host_software_installs SET uninstall_script_exit_code = 1 WHERE host_id = ? AND software_installer_id = ?`, - hostFailedUninstall.ID, si.InstallerID) - require.NoError(t, err) - return nil + _, _, err = ds.SetHostScriptExecutionResult(ctx, &fleet.HostScriptResultPayload{ + HostID: hostFailedUninstall.ID, + ExecutionID: execID, + ExitCode: 1, }) + require.NoError(t, err) // Host with successful uninstall tag = "-uninstalled" @@ -367,15 +401,15 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { TeamID: teamID, }) require.NoError(t, err) - err = ds.InsertSoftwareUninstallRequest(ctx, "uuid"+tag+tc, hostUninstalled.ID, si.InstallerID) + execID = "uuid" + tag + tc + err = ds.InsertSoftwareUninstallRequest(ctx, execID, hostUninstalled.ID, si.InstallerID) require.NoError(t, err) - ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - _, err = q.ExecContext(ctx, ` - UPDATE host_software_installs SET uninstall_script_exit_code = 0 WHERE host_id = ? AND software_installer_id = ?`, - hostUninstalled.ID, si.InstallerID) - require.NoError(t, err) - return nil + _, _, err = ds.SetHostScriptExecutionResult(ctx, &fleet.HostScriptResultPayload{ + HostID: hostUninstalled.ID, + ExecutionID: execID, + ExitCode: 0, }) + require.NoError(t, err) // Uninstall request with unknown host err = ds.InsertSoftwareUninstallRequest(ctx, "uuid"+tag+tc, 99999, si.InstallerID) @@ -385,16 +419,33 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { User: &fleet.User{GlobalRole: ptr.String("admin")}, } + // for this test, teamID is nil for no-team, but the ListHosts filter + // returns "all teams" if TeamFilter = nil, it needs to use TeamFilter = + // 0 for "no team" only. + teamFilter := teamID + if teamFilter == nil { + teamFilter = ptr.Uint(0) + } + // list hosts with software install pending requests expectStatus := fleet.SoftwareInstallPending hosts, err := ds.ListHosts(ctx, userTeamFilter, fleet.HostListOptions{ ListOptions: fleet.ListOptions{PerPage: 100}, SoftwareTitleIDFilter: installerMeta.TitleID, SoftwareStatusFilter: &expectStatus, - TeamFilter: teamID, + TeamFilter: teamFilter, }) require.NoError(t, err) - require.Len(t, hosts, 1) + + // get the names of hosts, useful for debugging + getHostNames := func(hosts []*fleet.Host) []string { + hostNames := make([]string, len(hosts)) + for _, h := range hosts { + hostNames = append(hostNames, h.Hostname) + } + return hostNames + } + require.Len(t, hosts, 1, getHostNames(hosts)) require.Equal(t, hostPendingInstall.ID, hosts[0].ID) // list hosts with all pending requests @@ -403,10 +454,10 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { ListOptions: fleet.ListOptions{PerPage: 100}, SoftwareTitleIDFilter: installerMeta.TitleID, SoftwareStatusFilter: &expectStatus, - TeamFilter: teamID, + TeamFilter: teamFilter, }) require.NoError(t, err) - require.Len(t, hosts, 2) + require.Len(t, hosts, 2, getHostNames(hosts)) assert.ElementsMatch(t, []uint{hostPendingInstall.ID, hostPendingUninstall.ID}, []uint{hosts[0].ID, hosts[1].ID}) // list hosts with software install failed requests @@ -415,10 +466,10 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { ListOptions: fleet.ListOptions{PerPage: 100}, SoftwareTitleIDFilter: installerMeta.TitleID, SoftwareStatusFilter: &expectStatus, - TeamFilter: teamID, + TeamFilter: teamFilter, }) require.NoError(t, err) - require.Len(t, hosts, 1) + require.Len(t, hosts, 1, getHostNames(hosts)) assert.ElementsMatch(t, []uint{hostFailedInstall.ID}, []uint{hosts[0].ID}) // list hosts with all failed requests @@ -427,10 +478,10 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { ListOptions: fleet.ListOptions{PerPage: 100}, SoftwareTitleIDFilter: installerMeta.TitleID, SoftwareStatusFilter: &expectStatus, - TeamFilter: teamID, + TeamFilter: teamFilter, }) require.NoError(t, err) - require.Len(t, hosts, 2) + require.Len(t, hosts, 2, getHostNames(hosts)) assert.ElementsMatch(t, []uint{hostFailedInstall.ID, hostFailedUninstall.ID}, []uint{hosts[0].ID, hosts[1].ID}) // list hosts with software installed @@ -439,10 +490,10 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { ListOptions: fleet.ListOptions{PerPage: 100}, SoftwareTitleIDFilter: installerMeta.TitleID, SoftwareStatusFilter: &expectStatus, - TeamFilter: teamID, + TeamFilter: teamFilter, }) require.NoError(t, err) - require.Len(t, hosts, 1) + require.Len(t, hosts, 1, getHostNames(hosts)) assert.ElementsMatch(t, []uint{hostInstalled.ID}, []uint{hosts[0].ID}) // list hosts with pending software uninstall requests @@ -451,10 +502,10 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { ListOptions: fleet.ListOptions{PerPage: 100}, SoftwareTitleIDFilter: installerMeta.TitleID, SoftwareStatusFilter: &expectStatus, - TeamFilter: teamID, + TeamFilter: teamFilter, }) require.NoError(t, err) - require.Len(t, hosts, 1) + require.Len(t, hosts, 1, getHostNames(hosts)) assert.ElementsMatch(t, []uint{hostPendingUninstall.ID}, []uint{hosts[0].ID}) // list hosts with failed software uninstall requests @@ -463,22 +514,21 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { ListOptions: fleet.ListOptions{PerPage: 100}, SoftwareTitleIDFilter: installerMeta.TitleID, SoftwareStatusFilter: &expectStatus, - TeamFilter: teamID, + TeamFilter: teamFilter, }) require.NoError(t, err) - require.Len(t, hosts, 1) + require.Len(t, hosts, 1, getHostNames(hosts)) assert.ElementsMatch(t, []uint{hostFailedUninstall.ID}, []uint{hosts[0].ID}) // list all hosts with the software title that shows up in host_software (after fleetd software query is run) hosts, err = ds.ListHosts(ctx, userTeamFilter, fleet.HostListOptions{ ListOptions: fleet.ListOptions{PerPage: 100}, SoftwareTitleIDFilter: installerMeta.TitleID, - TeamFilter: teamID, + TeamFilter: teamFilter, }) require.NoError(t, err) assert.Empty(t, hosts) - // get software title includes status summary, err := ds.GetSummaryHostSoftwareInstalls(ctx, installerMeta.InstallerID) require.NoError(t, err) require.Equal(t, fleet.SoftwareInstallerStatusSummary{ @@ -564,7 +614,7 @@ func testGetSoftwareInstallResult(t *testing.T, ds *Datastore) { require.NoError(t, err) beforeInstallRequest := time.Now() - installUUID, err := ds.InsertSoftwareInstallRequest(ctx, host.ID, installerID, false, nil) + installUUID, err := ds.InsertSoftwareInstallRequest(ctx, host.ID, installerID, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) res, err := ds.GetSoftwareInstallResults(ctx, installUUID) @@ -707,11 +757,17 @@ func testCleanupUnusedSoftwareInstallers(t *testing.T, ds *Datastore) { func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { ctx := context.Background() + t.Cleanup(func() { ds.testActivateSpecificNextActivities = nil }) // create a team team, err := ds.NewTeam(ctx, &fleet.Team{Name: t.Name()}) require.NoError(t, err) + // create a couple hosts + host1 := test.NewHost(t, ds, "host1", "1", "host1key", "host1uuid", time.Now()) + host2 := test.NewHost(t, ds, "host2", "2", "host2key", "host2uuid", time.Now()) + err = ds.AddHostsToTeam(ctx, &team.ID, []uint{host1.ID, host2.ID}) + require.NoError(t, err) user1 := test.NewUser(t, ds, "Alice", "alice@example.com", true) // TODO(roberto): perform better assertions, we should have everything @@ -950,6 +1006,102 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { {Name: ins1, Source: "apps", Browser: ""}, }) + instDetails1, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, &team.ID, *softwareInstallers[0].TitleID, false) + require.NoError(t, err) + + // add pending and completed installs for ins1 + _, err = ds.InsertSoftwareInstallRequest(ctx, host1.ID, instDetails1.InstallerID, fleet.HostSoftwareInstallOptions{}) + require.NoError(t, err) + execID2, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, instDetails1.InstallerID, fleet.HostSoftwareInstallOptions{}) + require.NoError(t, err) + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host2.ID, + InstallUUID: execID2, + InstallScriptExitCode: ptr.Int(0), + }) + require.NoError(t, err) + + summary, err := ds.GetSummaryHostSoftwareInstalls(ctx, instDetails1.InstallerID) + require.NoError(t, err) + require.Equal(t, fleet.SoftwareInstallerStatusSummary{Installed: 1, PendingInstall: 1}, *summary) + + // batch-set without changes + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{ + { + InstallScript: "install", + PostInstallScript: "post-install", + InstallerFile: tfr1, + StorageID: ins1, + Filename: ins1, + Title: ins1, + Source: "apps", + Version: "2", + PreInstallQuery: "select 1 from bar;", + UserID: user1.ID, + ValidatedLabels: &fleet.LabelIdentsWithScope{}, + }, + }) + require.NoError(t, err) + + // installs stats haven't changed + summary, err = ds.GetSummaryHostSoftwareInstalls(ctx, instDetails1.InstallerID) + require.NoError(t, err) + require.Equal(t, fleet.SoftwareInstallerStatusSummary{Installed: 1, PendingInstall: 1}, *summary) + + // remove ins1 and add ins0 + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{ + { + InstallScript: "install", + InstallerFile: tfr0, + StorageID: ins0, + Filename: ins0, + Title: ins0, + Source: "apps", + Version: "1", + PreInstallQuery: "select 0 from foo;", + UserID: user1.ID, + Platform: "darwin", + URL: "https://example.com", + ValidatedLabels: &fleet.LabelIdentsWithScope{}, + }, + }) + require.NoError(t, err) + + // stats don't report anything about ins1 anymore + summary, err = ds.GetSummaryHostSoftwareInstalls(ctx, instDetails1.InstallerID) + require.NoError(t, err) + require.Equal(t, fleet.SoftwareInstallerStatusSummary{Installed: 0, PendingInstall: 0}, *summary) + pendingHost1, err := ds.ListPendingSoftwareInstalls(ctx, host1.ID) + require.NoError(t, err) + require.Empty(t, pendingHost1) + + // add pending and completed installs for ins0 + softwareInstallers, err = ds.GetSoftwareInstallers(ctx, team.ID) + require.NoError(t, err) + require.Len(t, softwareInstallers, 1) + instDetails0, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, &team.ID, *softwareInstallers[0].TitleID, false) + require.NoError(t, err) + + _, err = ds.InsertSoftwareInstallRequest(ctx, host1.ID, instDetails0.InstallerID, fleet.HostSoftwareInstallOptions{}) + require.NoError(t, err) + execID2b, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, instDetails0.InstallerID, fleet.HostSoftwareInstallOptions{}) + require.NoError(t, err) + + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host2.ID, + InstallUUID: execID2b, + InstallScriptExitCode: ptr.Int(1), + }) + require.NoError(t, err) + + pendingHost1, err = ds.ListPendingSoftwareInstalls(ctx, host1.ID) + require.NoError(t, err) + require.Len(t, pendingHost1, 1) + + summary, err = ds.GetSummaryHostSoftwareInstalls(ctx, instDetails0.InstallerID) + require.NoError(t, err) + require.Equal(t, fleet.SoftwareInstallerStatusSummary{FailedInstall: 1, PendingInstall: 1}, *summary) + // remove everything err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{}) require.NoError(t, err) @@ -957,6 +1109,14 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { require.NoError(t, err) require.Empty(t, softwareInstallers) assertSoftware([]fleet.SoftwareTitle{}) + + // stats don't report anything about ins0 anymore + summary, err = ds.GetSummaryHostSoftwareInstalls(ctx, instDetails0.InstallerID) + require.NoError(t, err) + require.Equal(t, fleet.SoftwareInstallerStatusSummary{FailedInstall: 0, PendingInstall: 0}, *summary) + pendingHost1, err = ds.ListPendingSoftwareInstalls(ctx, host1.ID) + require.NoError(t, err) + require.Empty(t, pendingHost1) } func testGetSoftwareInstallerMetadataByTeamAndTitleID(t *testing.T, ds *Datastore) { @@ -1262,7 +1422,7 @@ func testDeletePendingSoftwareInstallsForPolicy(t *testing.T, ds *Datastore) { var count int // install for correct policy & correct status - executionID, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID1, false, &policy1.ID) + executionID, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID1, fleet.HostSoftwareInstallOptions{PolicyID: &policy1.ID}) require.NoError(t, err) err = sqlx.GetContext(ctx, ds.reader(ctx), &count, hostSoftwareInstallsCount, fleet.SoftwareInstallPending, executionID) @@ -1277,7 +1437,7 @@ func testDeletePendingSoftwareInstallsForPolicy(t *testing.T, ds *Datastore) { require.Equal(t, 0, count) // install for different policy & correct status - executionID, err = ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID2, false, &policy2.ID) + executionID, err = ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID2, fleet.HostSoftwareInstallOptions{PolicyID: &policy2.ID}) require.NoError(t, err) err = sqlx.GetContext(ctx, ds.reader(ctx), &count, hostSoftwareInstallsCount, fleet.SoftwareInstallPending, executionID) @@ -1292,7 +1452,7 @@ func testDeletePendingSoftwareInstallsForPolicy(t *testing.T, ds *Datastore) { require.Equal(t, 1, count) // install for correct policy & incorrect status - executionID, err = ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID1, false, &policy1.ID) + executionID, err = ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID1, fleet.HostSoftwareInstallOptions{PolicyID: &policy1.ID}) require.NoError(t, err) err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ @@ -1366,7 +1526,7 @@ func testGetHostLastInstallData(t *testing.T, ds *Datastore) { require.Nil(t, host1LastInstall) // Install installer.pkg on host1. - installUUID1, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, softwareInstallerID1, false, nil) + installUUID1, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, softwareInstallerID1, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) require.NotEmpty(t, installUUID1) @@ -1396,7 +1556,7 @@ func testGetHostLastInstallData(t *testing.T, ds *Datastore) { require.Equal(t, fleet.SoftwareInstalled, *host1LastInstall.Status) // Install installer2.pkg on host1. - installUUID2, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, softwareInstallerID2, false, nil) + installUUID2, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, softwareInstallerID2, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) require.NotEmpty(t, installUUID2) @@ -1416,7 +1576,7 @@ func testGetHostLastInstallData(t *testing.T, ds *Datastore) { require.Equal(t, fleet.SoftwareInstallPending, *host1LastInstall.Status) // Perform another installation of installer1.pkg. - installUUID3, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, softwareInstallerID1, false, nil) + installUUID3, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, softwareInstallerID1, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) require.NotEmpty(t, installUUID3) @@ -1428,7 +1588,16 @@ func testGetHostLastInstallData(t *testing.T, ds *Datastore) { require.NotNil(t, host1LastInstall.Status) require.Equal(t, fleet.SoftwareInstallPending, *host1LastInstall.Status) - // Set result of last installer1.pkg installation. + // Set result of last installer1.pkg installation, but first we need to set a + // result for installUUID2 so that this last installer1.pkg request is + // activated. + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host1.ID, + InstallUUID: installUUID2, + + InstallScriptExitCode: ptr.Int(0), + }) + require.NoError(t, err) err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ HostID: host1.ID, InstallUUID: installUUID3, @@ -1715,7 +1884,7 @@ func testBatchSetSoftwareInstallersScopedViaLabels(t *testing.T, ds *Datastore) globalOrTeamID, payload.Installer.Title, payload.Installer.Source) return err }) - _, err = ds.InsertSoftwareInstallRequest(ctx, host.ID, swID, false, nil) + _, err = ds.InsertSoftwareInstallRequest(ctx, host.ID, swID, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) installerIDs[i] = swID } @@ -2023,3 +2192,99 @@ Software won't be installed on Linux hosts with Debian-based distributions becau require.Len(t, team3Policies, 3) require.Equal(t, "[Install software] Something2 (msi) 3", team3Policies[2].Name) } + +func testGetSoftwareTitleNameFromExecutionID(t *testing.T, ds *Datastore) { + ctx := context.Background() + + user := test.NewUser(t, ds, "Alice", "alice@example.com", true) + host := test.NewHost(t, ds, "host1", "1", "host1key", "host1uuid", time.Now()) + + tfr1, err := fleet.NewTempFileReader(strings.NewReader("hello"), t.TempDir) + require.NoError(t, err) + + // create a couple software titles + installer1, _, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallerFile: tfr1, + BundleIdentifier: "foobar0", + Extension: "pkg", + StorageID: "storage0", + Filename: "foobar0", + Title: "foobar", + Version: "1.0", + Source: "apps", + UserID: user.ID, + ValidatedLabels: &fleet.LabelIdentsWithScope{}, + }) + require.NoError(t, err) + + installer2, _, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallerFile: tfr1, + BundleIdentifier: "foobar1", + Extension: "pkg", + StorageID: "storage1", + Filename: "foobar1", + Title: "barfoo", + Version: "1.0", + Source: "apps", + UserID: user.ID, + ValidatedLabels: &fleet.LabelIdentsWithScope{}, + }) + require.NoError(t, err) + + // get software title for unknown exec id + title, err := ds.GetSoftwareTitleNameFromExecutionID(ctx, "unknown") + require.ErrorIs(t, err, sql.ErrNoRows) + require.Empty(t, title) + + // create a couple pending software install request, the first will be + // immediately present in host_software_installs too (activated) + req1, err := ds.InsertSoftwareInstallRequest(ctx, host.ID, installer1, fleet.HostSoftwareInstallOptions{}) + require.NoError(t, err) + req2, err := ds.InsertSoftwareInstallRequest(ctx, host.ID, installer2, fleet.HostSoftwareInstallOptions{}) + require.NoError(t, err) + + title, err = ds.GetSoftwareTitleNameFromExecutionID(ctx, req1) + require.NoError(t, err) + require.Equal(t, "foobar", title) + + title, err = ds.GetSoftwareTitleNameFromExecutionID(ctx, req2) + require.NoError(t, err) + require.Equal(t, "barfoo", title) + + // record a result for req1, will be deleted from upcoming_activities + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host.ID, + InstallUUID: req1, + InstallScriptExitCode: ptr.Int(0), + }) + require.NoError(t, err) + + title, err = ds.GetSoftwareTitleNameFromExecutionID(ctx, req1) + require.NoError(t, err) + require.Equal(t, "foobar", title) + + title, err = ds.GetSoftwareTitleNameFromExecutionID(ctx, req2) + require.NoError(t, err) + require.Equal(t, "barfoo", title) + + // create an uninstall request for installer1 + req3 := uuid.NewString() + err = ds.InsertSoftwareUninstallRequest(ctx, req3, host.ID, installer1) + require.NoError(t, err) + + title, err = ds.GetSoftwareTitleNameFromExecutionID(ctx, req3) + require.NoError(t, err) + require.Equal(t, "foobar", title) + + // record a result for req2, will activate req3 so it is now in host_software_installs too + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host.ID, + InstallUUID: req2, + InstallScriptExitCode: ptr.Int(0), + }) + require.NoError(t, err) + + title, err = ds.GetSoftwareTitleNameFromExecutionID(ctx, req3) + require.NoError(t, err) + require.Equal(t, "foobar", title) +} diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index d654a9e0ad4f..a1f3925bdd23 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -3872,18 +3872,18 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { // create an installation request for vpp1 and vpp2, leaving vpp3 as // available only - vpp1CmdUUID := createVPPAppInstallRequest(t, ds, host, vpp1, user.ID) - vpp2CmdUUID := createVPPAppInstallRequest(t, ds, host, vpp2, user.ID) + vpp1CmdUUID := createVPPAppInstallRequest(t, ds, host, vpp1, user) + vpp2CmdUUID := createVPPAppInstallRequest(t, ds, host, vpp2, user) // make vpp1 install a success, while vpp2 has its initial request as failed // and a subsequent request as pending. createVPPAppInstallResult(t, ds, host, vpp1CmdUUID, fleet.MDMAppleStatusAcknowledged) createVPPAppInstallResult(t, ds, host, vpp2CmdUUID, fleet.MDMAppleStatusError) time.Sleep(time.Second) // ensure a different created_at timestamp - vpp2bCmdUUID := createVPPAppInstallRequest(t, ds, host, vpp2, user.ID) + vpp2bCmdUUID := createVPPAppInstallRequest(t, ds, host, vpp2, user) require.NotEmpty(t, vpp2bCmdUUID) // add an install request for the team host on vpp1, should not impact // main host - vpp1TmCmdUUID := createVPPAppInstallRequest(t, ds, tmHost, vpp1, user.ID) + vpp1TmCmdUUID := createVPPAppInstallRequest(t, ds, tmHost, vpp1, user) require.NotEmpty(t, vpp1TmCmdUUID) expected["vpp1apps"] = fleet.HostSoftwareWithInstaller{ @@ -4375,14 +4375,14 @@ func testListIOSHostSoftware(t *testing.T, ds *Datastore) { // create an installation request for vpp1 and vpp2, leaving vpp3 and vpp4 as // available only - vpp1CmdUUID := createVPPAppInstallRequest(t, ds, host, vpp1, user.ID) - vpp2CmdUUID := createVPPAppInstallRequest(t, ds, host, vpp2, user.ID) + vpp1CmdUUID := createVPPAppInstallRequest(t, ds, host, vpp1, user) + vpp2CmdUUID := createVPPAppInstallRequest(t, ds, host, vpp2, user) // make vpp1 install a success, while vpp2 has its initial request as failed // and a subsequent request as pending. createVPPAppInstallResult(t, ds, host, vpp1CmdUUID, fleet.MDMAppleStatusAcknowledged) createVPPAppInstallResult(t, ds, host, vpp2CmdUUID, fleet.MDMAppleStatusError) time.Sleep(time.Second) // ensure a different created_at timestamp - vpp2bCmdUUID := createVPPAppInstallRequest(t, ds, host, vpp2, user.ID) + vpp2bCmdUUID := createVPPAppInstallRequest(t, ds, host, vpp2, user) require.NotEmpty(t, vpp2bCmdUUID) expected["vpp1ios_apps"] = fleet.HostSoftwareWithInstaller{ @@ -4630,7 +4630,7 @@ func testListHostSoftwareInstallThenTransferTeam(t *testing.T, ds *Datastore) { require.NoError(t, err) // install it on the host - hostInstall1, err := ds.InsertSoftwareInstallRequest(ctx, host.ID, installerTm1, false, nil) + hostInstall1, err := ds.InsertSoftwareInstallRequest(ctx, host.ID, installerTm1, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ HostID: host.ID, @@ -4648,7 +4648,7 @@ func testListHostSoftwareInstallThenTransferTeam(t *testing.T, ds *Datastore) { require.NoError(t, err) // fail to install it on the host - vpp1CmdUUID := createVPPAppInstallRequest(t, ds, host, vppTm1.AdamID, user.ID) + vpp1CmdUUID := createVPPAppInstallRequest(t, ds, host, vppTm1.AdamID, user) createVPPAppInstallResult(t, ds, host, vpp1CmdUUID, fleet.MDMAppleStatusError) // add the successful installer to the reported installed software @@ -4743,7 +4743,7 @@ func testListHostSoftwareInstallThenDeleteInstallers(t *testing.T, ds *Datastore require.NoError(t, err) // fail to install it on the host - hostInstall1, err := ds.InsertSoftwareInstallRequest(ctx, host.ID, installerTm1, false, nil) + hostInstall1, err := ds.InsertSoftwareInstallRequest(ctx, host.ID, installerTm1, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ HostID: host.ID, @@ -4761,7 +4761,7 @@ func testListHostSoftwareInstallThenDeleteInstallers(t *testing.T, ds *Datastore require.NoError(t, err) // install it on the host - vpp1CmdUUID := createVPPAppInstallRequest(t, ds, host, vppTm1.AdamID, user.ID) + vpp1CmdUUID := createVPPAppInstallRequest(t, ds, host, vppTm1.AdamID, user) createVPPAppInstallResult(t, ds, host, vpp1CmdUUID, fleet.MDMAppleStatusAcknowledged) // add the successful VPP app to the reported installed software diff --git a/server/datastore/mysql/software_titles_test.go b/server/datastore/mysql/software_titles_test.go index 585c695f81ec..58b652ca680a 100644 --- a/server/datastore/mysql/software_titles_test.go +++ b/server/datastore/mysql/software_titles_test.go @@ -325,7 +325,7 @@ func testOrderSoftwareTitles(t *testing.T, ds *Datastore) { ValidatedLabels: &fleet.LabelIdentsWithScope{}, }) require.NoError(t, err) - _, err = ds.InsertSoftwareInstallRequest(ctx, host1.ID, installer2, false, nil) + _, err = ds.InsertSoftwareInstallRequest(ctx, host1.ID, installer2, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) test.CreateInsertGlobalVPPToken(t, ds) diff --git a/server/datastore/mysql/vpp.go b/server/datastore/mysql/vpp.go index 56b18ede3b31..fa598bef425e 100644 --- a/server/datastore/mysql/vpp.go +++ b/server/datastore/mysql/vpp.go @@ -67,34 +67,88 @@ func (ds *Datastore) GetSummaryHostVPPAppInstalls(ctx context.Context, teamID *u ) { var dest fleet.VPPAppStatusSummary - stmt := fmt.Sprintf(` + // TODO(sarah): do we need to handle host_deleted_at similar to GetSummaryHostSoftwareInstalls? + // Currently there is no host_deleted_at in host_vpp_software_installs, so + // not handling it as part of the unified queue work. + + // TODO(uniq): refactor vppHostStatusNamedQuery to use the same logic as below + + // TODO(uniq): AFAICT we don't have uniqueness for host_id + title_id in upcoming or + // past activities. In the past the max(id) approach was "good enough" as a proxy for the most + // recent activity since we didn't really need to worry about the order of activities. + // Moving to a time-based approach would be more accurate but would require a more complex and + // potentially slower query. + + stmt := ` +WITH + +-- select most recent upcoming activities for each host +upcoming AS ( + SELECT + max(ua.id) AS upcoming_id, -- ensure we use only the most recent attempt for each host + host_id + FROM + upcoming_activities ua + JOIN vpp_app_upcoming_activities vua ON ua.id = vua.upcoming_activity_id + JOIN hosts h ON host_id = h.id + WHERE + activity_type = 'vpp_app_install' + AND adam_id = :adam_id + AND vua.platform = :platform + AND (h.team_id = :team_id OR (h.team_id IS NULL AND :team_id = 0)) + GROUP BY + host_id +), + +-- select most recent past activities for each host +past AS ( + SELECT + max(hvsi.id) AS past_id, -- ensure we use only the most recent attempt for each host + host_id + FROM + host_vpp_software_installs hvsi + JOIN hosts h ON host_id = h.id + WHERE + adam_id = :adam_id + AND hvsi.platform = :platform + AND (h.team_id = :team_id OR (h.team_id IS NULL AND :team_id = 0)) + AND host_id NOT IN(SELECT host_id FROM upcoming) -- antijoin to exclude hosts with upcoming activities + AND hvsi.removed = 0 + GROUP BY + host_id +) + +-- count each status SELECT COALESCE(SUM( IF(status = :software_status_pending, 1, 0)), 0) AS pending, COALESCE(SUM( IF(status = :software_status_failed, 1, 0)), 0) AS failed, COALESCE(SUM( IF(status = :software_status_installed, 1, 0)), 0) AS installed FROM ( + +-- union most recent past and upcoming activities after joining to get statuses for most recent activities SELECT - %s + past.host_id, + CASE + WHEN ncr.status = :mdm_status_acknowledged THEN + :software_status_installed + WHEN ncr.status = :mdm_status_error OR ncr.status = :mdm_status_format_error THEN + :software_status_failed + ELSE + NULL -- either pending or not installed via VPP App + END AS status FROM - host_vpp_software_installs hvsi -INNER JOIN - hosts h ON hvsi.host_id = h.id -LEFT OUTER JOIN - nano_command_results ncr ON ncr.id = h.uuid AND ncr.command_uuid = hvsi.command_uuid -WHERE - hvsi.adam_id = :adam_id AND hvsi.platform = :platform AND - (h.team_id = :team_id OR (h.team_id IS NULL AND :team_id = 0)) AND - hvsi.id IN ( - SELECT - max(hvsi2.id) -- ensure we use only the most recently created install attempt for each host - FROM - host_vpp_software_installs hvsi2 - WHERE - hvsi2.adam_id = :adam_id AND hvsi2.platform = :platform - GROUP BY - hvsi2.host_id - ) -) s`, vppAppHostStatusNamedQuery("hvsi", "ncr", "status")) + past + JOIN host_vpp_software_installs hvsi ON hvsi.id = past_id + JOIN hosts h ON h.id = past.host_id + JOIN nano_command_results ncr ON ncr.id = h.uuid AND ncr.command_uuid = hvsi.command_uuid +UNION +SELECT + upcoming.host_id, + :software_status_pending AS status + FROM + upcoming + JOIN vpp_app_upcoming_activities vua ON upcoming_id = vua.upcoming_activity_id + JOIN upcoming_activities ua ON ua.id = upcoming_id) t` var tmID uint if teamID != nil { @@ -565,26 +619,78 @@ WHERE vat.global_or_team_id = ? AND va.title_id = ? } func (ds *Datastore) InsertHostVPPSoftwareInstall(ctx context.Context, hostID uint, appID fleet.VPPAppID, - commandUUID, associatedEventID string, selfService bool, policyID *uint, + commandUUID, associatedEventID string, opts fleet.HostSoftwareInstallOptions, ) error { - stmt := ` -INSERT INTO host_vpp_software_installs - (host_id, adam_id, platform, command_uuid, user_id, associated_event_id, self_service, policy_id) + const ( + insertUAStmt = ` +INSERT INTO upcoming_activities + (host_id, priority, user_id, fleet_initiated, activity_type, execution_id, payload) VALUES - (?,?,?,?,?,?,?,?) - ` + (?, ?, ?, ?, 'vpp_app_install', ?, + JSON_OBJECT( + 'self_service', ?, + 'associated_event_id', ?, + 'user', (SELECT JSON_OBJECT('name', name, 'email', email, 'gravatar_url', gravatar_url) FROM users WHERE id = ?) + ) + )` + + insertVAUAStmt = ` +INSERT INTO vpp_app_upcoming_activities + (upcoming_activity_id, adam_id, platform, policy_id) +VALUES + (?, ?, ?, ?)` + + hostExistsStmt = `SELECT 1 FROM hosts WHERE id = ?` + ) + + // we need to explicitly do this check here because we can't set a FK constraint on the schema + var hostExists bool + err := sqlx.GetContext(ctx, ds.reader(ctx), &hostExists, hostExistsStmt, hostID) + if err != nil { + if err == sql.ErrNoRows { + return notFound("Host").WithID(hostID) + } + + return ctxerr.Wrap(ctx, err, "checking if host exists") + } var userID *uint - if ctxUser := authz.UserFromContext(ctx); ctxUser != nil && policyID == nil { + if ctxUser := authz.UserFromContext(ctx); ctxUser != nil && opts.PolicyID == nil { userID = &ctxUser.ID } - if _, err := ds.writer(ctx).ExecContext(ctx, stmt, hostID, appID.AdamID, appID.Platform, commandUUID, userID, - associatedEventID, selfService, policyID); err != nil { - return ctxerr.Wrap(ctx, err, "insert into host_vpp_software_installs") - } + err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + res, err := tx.ExecContext(ctx, insertUAStmt, + hostID, + opts.Priority(), + userID, + opts.IsFleetInitiated(), + commandUUID, + opts.SelfService, + associatedEventID, + userID, + ) + if err != nil { + return ctxerr.Wrap(ctx, err, "insert vpp install request") + } - return nil + activityID, _ := res.LastInsertId() + _, err = tx.ExecContext(ctx, insertVAUAStmt, + activityID, + appID.AdamID, + appID.Platform, + opts.PolicyID, + ) + if err != nil { + return ctxerr.Wrap(ctx, err, "insert vpp install request join table") + } + + if _, err := ds.activateNextUpcomingActivity(ctx, tx, hostID, ""); err != nil { + return ctxerr.Wrap(ctx, err, "activate next activity") + } + return nil + }) + return err } func (ds *Datastore) MapAdamIDsPendingInstall(ctx context.Context, hostID uint) (map[string]struct{}, error) { diff --git a/server/datastore/mysql/vpp_test.go b/server/datastore/mysql/vpp_test.go index 87d3764a7fb1..b47666452f24 100644 --- a/server/datastore/mysql/vpp_test.go +++ b/server/datastore/mysql/vpp_test.go @@ -9,6 +9,7 @@ import ( "github.com/fleetdm/fleet/v4/server/contexts/viewer" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/mdm" + nanomdm_mysql "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/storage/mysql" "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/test" "github.com/google/uuid" @@ -332,7 +333,7 @@ func testVPPAppStatus(t *testing.T, ds *Datastore) { require.NoError(t, err) // simulate an install request of vpp1 on h1 - cmd1 := createVPPAppInstallRequest(t, ds, h1, vpp1.AdamID, user.ID) + cmd1 := createVPPAppInstallRequest(t, ds, h1, vpp1.AdamID, user) summary, err = ds.GetSummaryHostVPPAppInstalls(ctx, nil, vpp1) require.NoError(t, err) @@ -347,8 +348,8 @@ func testVPPAppStatus(t *testing.T, ds *Datastore) { // create a new request for h1 that supercedes the failed on, and a request // for h2 with a successful result. - cmd2 := createVPPAppInstallRequest(t, ds, h1, vpp1.AdamID, user.ID) - cmd3 := createVPPAppInstallRequest(t, ds, h2, vpp1.AdamID, user.ID) + cmd2 := createVPPAppInstallRequest(t, ds, h1, vpp1.AdamID, user) + cmd3 := createVPPAppInstallRequest(t, ds, h2, vpp1.AdamID, user) createVPPAppInstallResult(t, ds, h2, cmd3, fleet.MDMAppleStatusAcknowledged) actUser, act, err := ds.GetPastActivityDataForVPPAppInstall(ctx, &mdm.CommandResults{CommandUUID: cmd3}) @@ -375,7 +376,7 @@ func testVPPAppStatus(t *testing.T, ds *Datastore) { require.Equal(t, &fleet.VPPAppStatusSummary{Pending: 0, Failed: 0, Installed: 0}, summary) // simulate a successful request for team app vpp2 on h3 - cmd4 := createVPPAppInstallRequest(t, ds, h3, vpp2.AdamID, user.ID) + cmd4 := createVPPAppInstallRequest(t, ds, h3, vpp2.AdamID, user) createVPPAppInstallResult(t, ds, h3, cmd4, fleet.MDMAppleStatusAcknowledged) summary, err = ds.GetSummaryHostVPPAppInstalls(ctx, &team1.ID, vpp2) @@ -384,11 +385,11 @@ func testVPPAppStatus(t *testing.T, ds *Datastore) { // simulate a successful, failed and pending request for app vpp3 on team // (h3) and no team (h1, h2) - cmd5 := createVPPAppInstallRequest(t, ds, h3, vpp3.AdamID, user.ID) + cmd5 := createVPPAppInstallRequest(t, ds, h3, vpp3.AdamID, user) createVPPAppInstallResult(t, ds, h3, cmd5, fleet.MDMAppleStatusAcknowledged) - cmd6 := createVPPAppInstallRequest(t, ds, h1, vpp3.AdamID, user.ID) + cmd6 := createVPPAppInstallRequest(t, ds, h1, vpp3.AdamID, user) createVPPAppInstallResult(t, ds, h1, cmd6, fleet.MDMAppleStatusCommandFormatError) - createVPPAppInstallRequest(t, ds, h2, vpp3.AdamID, user.ID) + createVPPAppInstallRequest(t, ds, h2, vpp3.AdamID, user) // for no team, it sees the failed and pending counts summary, err = ds.GetSummaryHostVPPAppInstalls(ctx, nil, vpp3) @@ -415,32 +416,44 @@ func testVPPAppStatus(t *testing.T, ds *Datastore) { } // simulates creating the VPP app install request on the host, returns the command UUID. -func createVPPAppInstallRequest(t *testing.T, ds *Datastore, host *fleet.Host, adamID string, userID uint) string { +func createVPPAppInstallRequest(t *testing.T, ds *Datastore, host *fleet.Host, adamID string, user *fleet.User) string { ctx := context.Background() + ctx = viewer.NewContext(ctx, viewer.Viewer{User: user}) cmdUUID := uuid.NewString() - appleCmd := createRawAppleCmd("ProfileList", cmdUUID) - commander, _ := createMDMAppleCommanderAndStorage(t, ds) - err := commander.EnqueueCommand(ctx, []string{host.UUID}, appleCmd) - require.NoError(t, err) + eventID := uuid.NewString() - ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - _, err := q.ExecContext(ctx, - `INSERT INTO host_vpp_software_installs (host_id, adam_id, platform, command_uuid, user_id) VALUES (?, ?, ?, ?, ?)`, - host.ID, adamID, host.Platform, cmdUUID, userID) - return err - }) + err := ds.InsertHostVPPSoftwareInstall(ctx, host.ID, fleet.VPPAppID{ + AdamID: adamID, + Platform: fleet.AppleDevicePlatform(host.Platform), + }, cmdUUID, eventID, fleet.HostSoftwareInstallOptions{}) + require.NoError(t, err) return cmdUUID } func createVPPAppInstallResult(t *testing.T, ds *Datastore, host *fleet.Host, cmdUUID string, status string) { ctx := context.Background() + ctx = context.WithValue(ctx, fleet.ActivityWebhookContextKey, true) - ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - _, err := q.ExecContext(ctx, `INSERT INTO nano_command_results (id, command_uuid, status, result) VALUES (?, ?, ?, '`), + } + err = nanoDB.StoreCommandReport(nanoCtx, cmdRes) + require.NoError(t, err) + + // inserting the activity is what marks the upcoming activity as completed + // (and activates the next one). + err = ds.NewActivity(ctx, nil, fleet.ActivityInstalledAppStoreApp{ + HostID: host.ID, + CommandUUID: cmdUUID, + }, []byte(`{}`), time.Now()) + require.NoError(t, err) } func testVPPApps(t *testing.T, ds *Datastore) { @@ -462,6 +475,7 @@ func testVPPApps(t *testing.T, ds *Datastore) { HardwareSerial: "654321a", }) require.NoError(t, err) + nanoEnrollAndSetHostMDMData(t, ds, h1, false) software := []fleet.Software{ {Name: "foo", Version: "0.0.1", BundleIdentifier: "b1"}, {Name: "foo", Version: "0.0.2", BundleIdentifier: "b1"}, @@ -495,7 +509,7 @@ func testVPPApps(t *testing.T, ds *Datastore) { _, err = ds.InsertVPPAppWithTeam(ctx, appNoTeam2, nil) require.NoError(t, err) - // Check that host_vpp_software_installs works + // Check that inserting pending vpp installs works u, err := ds.NewUser(ctx, &fleet.User{ Password: []byte("p4ssw0rd.123"), Name: "user1", @@ -504,37 +518,60 @@ func testVPPApps(t *testing.T, ds *Datastore) { }) require.NoError(t, err) ctx = viewer.NewContext(ctx, viewer.Viewer{User: u}) - err = ds.InsertHostVPPSoftwareInstall(ctx, 1, app1.VPPAppID, "a", "b", false, nil) - require.NoError(t, err) - err = ds.InsertHostVPPSoftwareInstall(ctx, 2, app2.VPPAppID, "c", "d", true, nil) + err = ds.InsertHostVPPSoftwareInstall(ctx, h1.ID, app1.VPPAppID, "a", "b", fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) - var results []struct { - HostID uint `db:"host_id"` - UserID *uint `db:"user_id"` - AdamID string `db:"adam_id"` - CommandUUID string `db:"command_uuid"` - AssociatedEventID string `db:"associated_event_id"` - SelfService bool `db:"self_service"` - } - err = sqlx.SelectContext(ctx, ds.reader(ctx), &results, `SELECT host_id, user_id, adam_id, command_uuid, associated_event_id, self_service FROM host_vpp_software_installs ORDER BY adam_id`) - require.NoError(t, err) - require.Len(t, results, 2) - a1 := results[0] - a2 := results[1] - require.Equal(t, a1.HostID, uint(1)) - require.Equal(t, a1.UserID, ptr.Uint(u.ID)) - require.Equal(t, a1.AdamID, app1.AdamID) - require.Equal(t, a1.CommandUUID, "a") - require.Equal(t, a1.AssociatedEventID, "b") - require.False(t, a1.SelfService) - require.Equal(t, a2.HostID, uint(2)) - require.Equal(t, a2.UserID, ptr.Uint(u.ID)) - require.Equal(t, a2.AdamID, app2.AdamID) - require.Equal(t, a2.CommandUUID, "c") - require.Equal(t, a2.AssociatedEventID, "d") - require.True(t, a2.SelfService) + // non-existing host + err = ds.InsertHostVPPSoftwareInstall(ctx, h1.ID+1, app2.VPPAppID, "c", "d", fleet.HostSoftwareInstallOptions{SelfService: true}) + require.Error(t, err) + var nfe fleet.NotFoundError + require.ErrorAs(t, err, &nfe) + + // create host 2 + h2, err := ds.NewHost(ctx, &fleet.Host{ + Hostname: "macos-test-2", + OsqueryHostID: ptr.String("osquery-macos-2"), + NodeKey: ptr.String("node-key-macos-2"), + UUID: uuid.NewString(), + Platform: "darwin", + HardwareSerial: "654321b", + }) + require.NoError(t, err) + nanoEnrollAndSetHostMDMData(t, ds, h2, false) + err = ds.InsertHostVPPSoftwareInstall(ctx, h2.ID, app2.VPPAppID, "c", "d", fleet.HostSoftwareInstallOptions{SelfService: true}) + require.NoError(t, err) + + acts, _, err := ds.ListHostUpcomingActivities(ctx, h1.ID, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, acts, 1) + require.NotNil(t, acts[0].ActorFullName) + require.Equal(t, u.Name, *acts[0].ActorFullName) + // app1 software title because it matched an existing software "foo" by bundle identifier + require.JSONEq(t, fmt.Sprintf(`{ + "app_store_id":"%s", + "command_uuid":"a", + "host_display_name":"%s", + "host_id":%d, + "self_service":false, + "software_title":"foo", + "status":"pending_install" + }`, app1.AdamID, h1.DisplayName(), h1.ID), string(*acts[0].Details)) + + acts, _, err = ds.ListHostUpcomingActivities(ctx, h2.ID, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, acts, 1) + require.NotNil(t, acts[0].ActorFullName) + require.Equal(t, u.Name, *acts[0].ActorFullName) + require.JSONEq(t, fmt.Sprintf(`{ + "app_store_id":"%s", + "command_uuid":"c", + "host_display_name":"%s", + "host_id":%d, + "self_service":true, + "software_title":"vpp_app_2", + "status":"pending_install" + }`, app2.AdamID, h2.DisplayName(), h2.ID), string(*acts[0].Details)) // Check that getting the assigned apps works appSet, err := ds.GetAssignedVPPApps(ctx, &team.ID) diff --git a/server/fleet/activities.go b/server/fleet/activities.go index 4d184426f7df..443370d9eb0c 100644 --- a/server/fleet/activities.go +++ b/server/fleet/activities.go @@ -12,6 +12,42 @@ type ContextKey string // ActivityWebhookContextKey is the context key to indicate that the activity webhook has been processed before saving the activity. const ActivityWebhookContextKey = ContextKey("ActivityWebhook") +type Activity struct { + CreateTimestamp + + // ID is the activity id in the activities table, it is omitted for upcoming + // activities as those are "virtual activities" generated from entries in + // queues (e.g. pending host_script_results). + ID uint `json:"id,omitempty" db:"id"` + + // UUID is the activity UUID for the upcoming activities, as identified in + // the relevant queue (e.g. pending host_script_results). It is omitted for + // past activities as those are "real activities" with an activity id. + UUID string `json:"uuid,omitempty" db:"uuid"` + + ActorFullName *string `json:"actor_full_name,omitempty" db:"name"` + ActorID *uint `json:"actor_id,omitempty" db:"user_id"` + ActorGravatar *string `json:"actor_gravatar,omitempty" db:"gravatar_url"` + ActorEmail *string `json:"actor_email,omitempty" db:"user_email"` + Type string `json:"type" db:"activity_type"` + Details *json.RawMessage `json:"details" db:"details"` + Streamed *bool `json:"-" db:"streamed"` +} + +// AuthzType implement AuthzTyper to be able to verify access to activities +func (*Activity) AuthzType() string { + return "activity" +} + +// UpcomingActivity is the augmented activity type used to return the list of +// upcoming (pending) activities for a host. +type UpcomingActivity struct { + Activity + + FleetInitiated bool `json:"fleet_initiated" db:"fleet_initiated"` + Cancellable bool `json:"cancellable" db:"cancellable"` +} + // ActivityDetailsList is used to generate documentation. var ActivityDetailsList = []ActivityDetails{ ActivityTypeCreatedPack{}, @@ -588,33 +624,6 @@ func (a ActivityTypeUserAddedBySSO) Documentation() (activity string, details st `This activity does not contain any detail fields.`, "" } -type Activity struct { - CreateTimestamp - - // ID is the activity id in the activities table, it is omitted for upcoming - // activities as those are "virtual activities" generated from entries in - // queues (e.g. pending host_script_results). - ID uint `json:"id,omitempty" db:"id"` - - // UUID is the activity UUID for the upcoming activities, as identified in - // the relevant queue (e.g. pending host_script_results). It is omitted for - // past activities as those are "real activities" with an activity id. - UUID string `json:"uuid,omitempty" db:"uuid"` - - ActorFullName *string `json:"actor_full_name,omitempty" db:"name"` - ActorID *uint `json:"actor_id,omitempty" db:"user_id"` - ActorGravatar *string `json:"actor_gravatar,omitempty" db:"gravatar_url"` - ActorEmail *string `json:"actor_email,omitempty" db:"user_email"` - Type string `json:"type" db:"activity_type"` - Details *json.RawMessage `json:"details" db:"details"` - Streamed *bool `json:"-" db:"streamed"` -} - -// AuthzType implement AuthzTyper to be able to verify access to activities -func (*Activity) AuthzType() string { - return "activity" -} - type ActivityTypeUserLoggedIn struct { PublicIP string `json:"public_ip"` } diff --git a/server/fleet/apple_mdm.go b/server/fleet/apple_mdm.go index 485988029ec1..0bfdaca1fe36 100644 --- a/server/fleet/apple_mdm.go +++ b/server/fleet/apple_mdm.go @@ -22,7 +22,6 @@ type MDMAppleCommandIssuer interface { DeviceLock(ctx context.Context, host *Host, uuid string) (unlockPIN string, err error) EraseDevice(ctx context.Context, host *Host, uuid string) error InstallEnterpriseApplication(ctx context.Context, hostUUIDs []string, uuid string, manifestURL string) error - InstallApplication(ctx context.Context, hostUUIDs []string, uuid string, adamID string) error DeviceConfigured(ctx context.Context, hostUUID, cmdUUID string) error } diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index e05965137fcd..2d812ba9da8d 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -555,7 +555,7 @@ type Datastore interface { // InsertSoftwareInstallRequest tracks a new request to install the provided // software installer in the host. It returns the auto-generated installation // uuid. - InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint, selfService bool, policyID *uint) (string, error) + InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint, opts HostSoftwareInstallOptions) (string, error) // InsertSoftwareUninstallRequest tracks a new request to uninstall the provided // software installer on the host. executionID is the script execution ID corresponding to uninstall script InsertSoftwareUninstallRequest(ctx context.Context, executionID string, hostID uint, softwareInstallerID uint) error @@ -667,7 +667,7 @@ type Datastore interface { NewActivity(ctx context.Context, user *User, activity ActivityDetails, details []byte, createdAt time.Time) error ListActivities(ctx context.Context, opt ListActivitiesOptions) ([]*Activity, *PaginationMetadata, error) MarkActivitiesAsStreamed(ctx context.Context, activityIDs []uint) error - ListHostUpcomingActivities(ctx context.Context, hostID uint, opt ListOptions) ([]*Activity, *PaginationMetadata, error) + ListHostUpcomingActivities(ctx context.Context, hostID uint, opt ListOptions) ([]*UpcomingActivity, *PaginationMetadata, error) ListHostPastActivities(ctx context.Context, hostID uint, opt ListOptions) ([]*Activity, *PaginationMetadata, error) IsExecutionPendingForHost(ctx context.Context, hostID uint, scriptID uint) (bool, error) @@ -1623,9 +1623,6 @@ type Datastore interface { // NewHostScriptExecutionRequest creates a new host script result entry with // just the script to run information (result is not yet available). NewHostScriptExecutionRequest(ctx context.Context, request *HostScriptRequestPayload) (*HostScriptResult, error) - // NewInternalScriptExecutionRequest creates a new host script result entry with - // just the script to run information (result is not yet available), with the script marked as internal. - NewInternalScriptExecutionRequest(ctx context.Context, request *HostScriptRequestPayload) (*HostScriptResult, error) // SetHostScriptExecutionResult stores the result of a host script execution // and returns the updated host script result record. Note that it does not // fail if the script execution request does not exist, in this case it will @@ -1640,6 +1637,10 @@ type Datastore interface { // to record a result. Pass onlyShowInternal as true to return only scripts that execute when script execution is // globally disabled (uninstall/lock/unlock/wipe). ListPendingHostScriptExecutions(ctx context.Context, hostID uint, onlyShowInternal bool) ([]*HostScriptResult, error) + // ListReadyToExecuteScriptsForHost is like ListPendingHostScriptExecutions + // except that it only returns those that are ready to execute ("activated" in + // the upcoming activities queue, available for orbit to process). + ListReadyToExecuteScriptsForHost(ctx context.Context, hostID uint, onlyShowInternal bool) ([]*HostScriptResult, error) // NewScript creates a new saved script. NewScript(ctx context.Context, script *Script) (*Script, error) @@ -1736,6 +1737,11 @@ type Datastore interface { // ListPendingSoftwareInstalls returns a list of software // installer execution IDs that have not yet been run for a given host ListPendingSoftwareInstalls(ctx context.Context, hostID uint) ([]string, error) + // ListReadyToExecuteSoftwareInstalls is like ListPendingSoftwareInstalls + // except that it only returns software installs that are ready to execute + // ("activated" in the upcoming activities queue, available for orbit to + // process). + ListReadyToExecuteSoftwareInstalls(ctx context.Context, hostID uint) ([]string, error) // GetHostLastInstallData returns the data for the last installation of a package on a host. GetHostLastInstallData(ctx context.Context, hostID, installerID uint) (*HostLastInstallData, error) @@ -1828,7 +1834,7 @@ type Datastore interface { SetTeamVPPApps(ctx context.Context, teamID *uint, appIDs []VPPAppTeam) error InsertVPPAppWithTeam(ctx context.Context, app *VPPApp, teamID *uint) (*VPPApp, error) - InsertHostVPPSoftwareInstall(ctx context.Context, hostID uint, appID VPPAppID, commandUUID, associatedEventID string, selfService bool, policyID *uint) error + InsertHostVPPSoftwareInstall(ctx context.Context, hostID uint, appID VPPAppID, commandUUID, associatedEventID string, opts HostSoftwareInstallOptions) error GetPastActivityDataForVPPAppInstall(ctx context.Context, commandResults *mdm.CommandResults) (*User, *ActivityInstalledAppStoreApp, error) GetVPPTokenByLocation(ctx context.Context, loc string) (*VPPTokenDB, error) diff --git a/server/fleet/scripts.go b/server/fleet/scripts.go index 666018d45bd4..6dcf22253705 100644 --- a/server/fleet/scripts.go +++ b/server/fleet/scripts.go @@ -97,16 +97,18 @@ type HostScriptExecution struct { // SetLastExecution updates the LastExecution field of the HostScriptDetail if the provided details // are more recent than the current LastExecution. It returns true if the LastExecution was updated. func (hs *HostScriptDetail) setLastExecution(executionID *string, executedAt *time.Time, exitCode *int64, hsrID *uint) bool { - if hsrID == nil || executionID == nil || executedAt == nil { + if executionID == nil || executedAt == nil { // no new execution, nothing to do return false } newHSE := &HostScriptExecution{ - HSRID: *hsrID, ExecutionID: *executionID, ExecutedAt: *executedAt, } + if hsrID != nil { + newHSE.HSRID = *hsrID + } switch { case exitCode == nil: newHSE.Status = "pending" @@ -154,6 +156,16 @@ type HostScriptRequestPayload struct { SetupExperienceScriptID *uint `json:"-"` } +// Priority returns the priority to assign to this activity in the upcoming +// activities queue. It is the default priority except when the script is part +// of the setup experience flow. +func (r HostScriptRequestPayload) Priority() int { + if r.SetupExperienceScriptID != nil { + return 100 + } + return 0 +} + func (r HostScriptRequestPayload) ValidateParams(waitForResult time.Duration) error { if r.ScriptContents == "" && r.ScriptID == nil && r.ScriptName == "" { return NewInvalidArgumentError("script", `One of 'script_id', 'script_contents', or 'script_name' is required.`) diff --git a/server/fleet/service.go b/server/fleet/service.go index bf7b833e1026..a246ff4c134f 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -37,7 +37,7 @@ type EnterpriseOverrides struct { MDMAppleEditedAppleOSUpdates func(ctx context.Context, teamID *uint, appleDevice AppleDevice, updates AppleOSUpdateSettings) error SetupExperienceNextStep func(ctx context.Context, hostUUID string) (bool, error) GetVPPTokenIfCanInstallVPPApps func(ctx context.Context, appleDevice bool, host *Host) (string, error) - InstallVPPAppPostValidation func(ctx context.Context, host *Host, vppApp *VPPApp, token string, selfService bool, policyID *uint) (string, error) + InstallVPPAppPostValidation func(ctx context.Context, host *Host, vppApp *VPPApp, token string, opts HostSoftwareInstallOptions) (string, error) } type OsqueryService interface { @@ -587,7 +587,7 @@ type Service interface { // host. Those are activities that are queued or scheduled to run on the host // but haven't run yet. It also returns the total (unpaginated) count of upcoming // activities. - ListHostUpcomingActivities(ctx context.Context, hostID uint, opt ListOptions) ([]*Activity, *PaginationMetadata, error) + ListHostUpcomingActivities(ctx context.Context, hostID uint, opt ListOptions) ([]*UpcomingActivity, *PaginationMetadata, error) // ListHostPastActivities lists the activities that have already happened for the specified host. ListHostPastActivities(ctx context.Context, hostID uint, opt ListOptions) ([]*Activity, *PaginationMetadata, error) @@ -657,7 +657,7 @@ type Service interface { GetVPPTokenIfCanInstallVPPApps(ctx context.Context, appleDevice bool, host *Host) (string, error) // InstallVPPAppPostValidation installs a VPP app, assuming that GetVPPTokenIfCanInstallVPPApps has passed and provided a VPP token - InstallVPPAppPostValidation(ctx context.Context, host *Host, vppApp *VPPApp, token string, selfService bool, policyID *uint) (string, error) + InstallVPPAppPostValidation(ctx context.Context, host *Host, vppApp *VPPApp, token string, opts HostSoftwareInstallOptions) (string, error) // UninstallSoftwareTitle uninstalls a software title in the given host. UninstallSoftwareTitle(ctx context.Context, hostID uint, softwareTitleID uint) error diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index 8ca8e9346ad5..70a9168e6d55 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -675,3 +675,28 @@ type SoftwareScopeLabel struct { Exclude bool `db:"exclude" json:"-"` // not rendered in JSON, used when processing LabelsIncludeAny and LabelsExcludeAny on parent title (may be the empty value in some cases) TitleID uint `db:"title_id" json:"-"` // not rendered in JSON, used to store the associated title ID (may be the empty value in some cases) } + +// HostSoftwareInstallOptions contains options that apply to a software or VPP +// app install request. +type HostSoftwareInstallOptions struct { + SelfService bool + PolicyID *uint + ForSetupExperience bool +} + +// IsFleetInitiated returns true if the software install is initiated by Fleet. +// Software installs initiated via a policy are fleet-initiated (and we also +// make sure SelfService is false, as this case is always user-initiated). +func (o HostSoftwareInstallOptions) IsFleetInitiated() bool { + return !o.SelfService && o.PolicyID != nil +} + +// Priority returns the upcoming activities queue priority to use for this +// software installation. Software installed for the setup experience is +// prioritized over other software installations. +func (o HostSoftwareInstallOptions) Priority() int { + if o.ForSetupExperience { + return 100 + } + return 0 +} diff --git a/server/mdm/apple/commander.go b/server/mdm/apple/commander.go index 15914ac393b7..82ff022bd09c 100644 --- a/server/mdm/apple/commander.go +++ b/server/mdm/apple/commander.go @@ -171,32 +171,6 @@ func (svc *MDMAppleCommander) EraseDevice(ctx context.Context, host *fleet.Host, return nil } -func (svc *MDMAppleCommander) InstallApplication(ctx context.Context, hostUUIDs []string, uuid string, adamID string) error { - raw := fmt.Sprintf(` - - - - Command - - ManagementFlags - 0 - Options - - PurchaseMethod - 1 - - RequestType - InstallApplication - iTunesStoreID - %s - - CommandUUID - %s - -`, adamID, uuid) - return svc.EnqueueCommand(ctx, hostUUIDs, raw) -} - func (svc *MDMAppleCommander) InstallEnterpriseApplication(ctx context.Context, hostUUIDs []string, uuid string, manifestURL string) error { raw := fmt.Sprintf(` diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index e5306955b41a..93ab6da6af34 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -129,8 +129,6 @@ type ListPacksForHostFunc func(ctx context.Context, hid uint) (packs []*fleet.Pa type ApplyLabelSpecsFunc func(ctx context.Context, specs []*fleet.LabelSpec) error -type UpdateLabelMembershipByHostIDsFunc func(ctx context.Context, labelID uint, hostIDs []uint) (err error) - type GetLabelSpecsFunc func(ctx context.Context) ([]*fleet.LabelSpec, error) type GetLabelSpecFunc func(ctx context.Context, name string) (*fleet.LabelSpec, error) @@ -139,6 +137,8 @@ type AddLabelsToHostFunc func(ctx context.Context, hostID uint, labelIDs []uint) type RemoveLabelsFromHostFunc func(ctx context.Context, hostID uint, labelIDs []uint) error +type UpdateLabelMembershipByHostIDsFunc func(ctx context.Context, labelID uint, hostIds []uint) (err error) + type NewLabelFunc func(ctx context.Context, Label *fleet.Label, opts ...fleet.OptionalArg) (*fleet.Label, error) type SaveLabelFunc func(ctx context.Context, label *fleet.Label, teamFilter fleet.TeamFilter) (*fleet.Label, []uint, error) @@ -415,7 +415,7 @@ type ListSoftwareTitlesFunc func(ctx context.Context, opt fleet.SoftwareTitleLis type SoftwareTitleByIDFunc func(ctx context.Context, id uint, teamID *uint, tmFilter fleet.TeamFilter) (*fleet.SoftwareTitle, error) -type InsertSoftwareInstallRequestFunc func(ctx context.Context, hostID uint, softwareInstallerID uint, selfService bool, policyID *uint) (string, error) +type InsertSoftwareInstallRequestFunc func(ctx context.Context, hostID uint, softwareInstallerID uint, opts fleet.HostSoftwareInstallOptions) (string, error) type InsertSoftwareUninstallRequestFunc func(ctx context.Context, executionID string, hostID uint, softwareInstallerID uint) error @@ -481,7 +481,7 @@ type ListActivitiesFunc func(ctx context.Context, opt fleet.ListActivitiesOption type MarkActivitiesAsStreamedFunc func(ctx context.Context, activityIDs []uint) error -type ListHostUpcomingActivitiesFunc func(ctx context.Context, hostID uint, opt fleet.ListOptions) ([]*fleet.Activity, *fleet.PaginationMetadata, error) +type ListHostUpcomingActivitiesFunc func(ctx context.Context, hostID uint, opt fleet.ListOptions) ([]*fleet.UpcomingActivity, *fleet.PaginationMetadata, error) type ListHostPastActivitiesFunc func(ctx context.Context, hostID uint, opt fleet.ListOptions) ([]*fleet.Activity, *fleet.PaginationMetadata, error) @@ -1039,14 +1039,14 @@ type SetOrUpdateMDMAppleDeclarationFunc func(ctx context.Context, declaration *f type NewHostScriptExecutionRequestFunc func(ctx context.Context, request *fleet.HostScriptRequestPayload) (*fleet.HostScriptResult, error) -type NewInternalScriptExecutionRequestFunc func(ctx context.Context, request *fleet.HostScriptRequestPayload) (*fleet.HostScriptResult, error) - type SetHostScriptExecutionResultFunc func(ctx context.Context, result *fleet.HostScriptResultPayload) (hsr *fleet.HostScriptResult, action string, err error) type GetHostScriptExecutionResultFunc func(ctx context.Context, execID string) (*fleet.HostScriptResult, error) type ListPendingHostScriptExecutionsFunc func(ctx context.Context, hostID uint, onlyShowInternal bool) ([]*fleet.HostScriptResult, error) +type ListReadyToExecuteScriptsForHostFunc func(ctx context.Context, hostID uint, onlyShowInternal bool) ([]*fleet.HostScriptResult, error) + type NewScriptFunc func(ctx context.Context, script *fleet.Script) (*fleet.Script, error) type ScriptFunc func(ctx context.Context, id uint) (*fleet.Script, error) @@ -1095,6 +1095,8 @@ type GetSoftwareInstallDetailsFunc func(ctx context.Context, executionId string) type ListPendingSoftwareInstallsFunc func(ctx context.Context, hostID uint) ([]string, error) +type ListReadyToExecuteSoftwareInstallsFunc func(ctx context.Context, hostID uint) ([]string, error) + type GetHostLastInstallDataFunc func(ctx context.Context, hostID uint, installerID uint) (*fleet.HostLastInstallData, error) type MatchOrCreateSoftwareInstallerFunc func(ctx context.Context, payload *fleet.UploadSoftwareInstallerPayload) (installerID uint, titleID uint, err error) @@ -1153,7 +1155,7 @@ type SetTeamVPPAppsFunc func(ctx context.Context, teamID *uint, appIDs []fleet.V type InsertVPPAppWithTeamFunc func(ctx context.Context, app *fleet.VPPApp, teamID *uint) (*fleet.VPPApp, error) -type InsertHostVPPSoftwareInstallFunc func(ctx context.Context, hostID uint, appID fleet.VPPAppID, commandUUID string, associatedEventID string, selfService bool, policyID *uint) error +type InsertHostVPPSoftwareInstallFunc func(ctx context.Context, hostID uint, appID fleet.VPPAppID, commandUUID string, associatedEventID string, opts fleet.HostSoftwareInstallOptions) error type GetPastActivityDataForVPPAppInstallFunc func(ctx context.Context, commandResults *mdm.CommandResults) (*fleet.User, *fleet.ActivityInstalledAppStoreApp, error) @@ -1372,9 +1374,6 @@ type DataStore struct { ApplyLabelSpecsFunc ApplyLabelSpecsFunc ApplyLabelSpecsFuncInvoked bool - UpdateLabelMembershipByHostIDsFunc UpdateLabelMembershipByHostIDsFunc - UpdateLabelMembershipByHostIDsFuncInvoked bool - GetLabelSpecsFunc GetLabelSpecsFunc GetLabelSpecsFuncInvoked bool @@ -1387,6 +1386,9 @@ type DataStore struct { RemoveLabelsFromHostFunc RemoveLabelsFromHostFunc RemoveLabelsFromHostFuncInvoked bool + UpdateLabelMembershipByHostIDsFunc UpdateLabelMembershipByHostIDsFunc + UpdateLabelMembershipByHostIDsFuncInvoked bool + NewLabelFunc NewLabelFunc NewLabelFuncInvoked bool @@ -2737,9 +2739,6 @@ type DataStore struct { NewHostScriptExecutionRequestFunc NewHostScriptExecutionRequestFunc NewHostScriptExecutionRequestFuncInvoked bool - NewInternalScriptExecutionRequestFunc NewInternalScriptExecutionRequestFunc - NewInternalScriptExecutionRequestFuncInvoked bool - SetHostScriptExecutionResultFunc SetHostScriptExecutionResultFunc SetHostScriptExecutionResultFuncInvoked bool @@ -2749,6 +2748,9 @@ type DataStore struct { ListPendingHostScriptExecutionsFunc ListPendingHostScriptExecutionsFunc ListPendingHostScriptExecutionsFuncInvoked bool + ListReadyToExecuteScriptsForHostFunc ListReadyToExecuteScriptsForHostFunc + ListReadyToExecuteScriptsForHostFuncInvoked bool + NewScriptFunc NewScriptFunc NewScriptFuncInvoked bool @@ -2821,6 +2823,9 @@ type DataStore struct { ListPendingSoftwareInstallsFunc ListPendingSoftwareInstallsFunc ListPendingSoftwareInstallsFuncInvoked bool + ListReadyToExecuteSoftwareInstallsFunc ListReadyToExecuteSoftwareInstallsFunc + ListReadyToExecuteSoftwareInstallsFuncInvoked bool + GetHostLastInstallDataFunc GetHostLastInstallDataFunc GetHostLastInstallDataFuncInvoked bool @@ -3373,13 +3378,6 @@ func (s *DataStore) ApplyLabelSpecs(ctx context.Context, specs []*fleet.LabelSpe return s.ApplyLabelSpecsFunc(ctx, specs) } -func (s *DataStore) UpdateLabelMembershipByHostIDs(ctx context.Context, labelID uint, hostIDs []uint) (err error) { - s.mu.Lock() - s.UpdateLabelMembershipByHostIDsFuncInvoked = true - s.mu.Unlock() - return s.UpdateLabelMembershipByHostIDsFunc(ctx, labelID, hostIDs) -} - func (s *DataStore) GetLabelSpecs(ctx context.Context) ([]*fleet.LabelSpec, error) { s.mu.Lock() s.GetLabelSpecsFuncInvoked = true @@ -3408,6 +3406,13 @@ func (s *DataStore) RemoveLabelsFromHost(ctx context.Context, hostID uint, label return s.RemoveLabelsFromHostFunc(ctx, hostID, labelIDs) } +func (s *DataStore) UpdateLabelMembershipByHostIDs(ctx context.Context, labelID uint, hostIds []uint) (err error) { + s.mu.Lock() + s.UpdateLabelMembershipByHostIDsFuncInvoked = true + s.mu.Unlock() + return s.UpdateLabelMembershipByHostIDsFunc(ctx, labelID, hostIds) +} + func (s *DataStore) NewLabel(ctx context.Context, Label *fleet.Label, opts ...fleet.OptionalArg) (*fleet.Label, error) { s.mu.Lock() s.NewLabelFuncInvoked = true @@ -4374,11 +4379,11 @@ func (s *DataStore) SoftwareTitleByID(ctx context.Context, id uint, teamID *uint return s.SoftwareTitleByIDFunc(ctx, id, teamID, tmFilter) } -func (s *DataStore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint, selfService bool, policyID *uint) (string, error) { +func (s *DataStore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint, opts fleet.HostSoftwareInstallOptions) (string, error) { s.mu.Lock() s.InsertSoftwareInstallRequestFuncInvoked = true s.mu.Unlock() - return s.InsertSoftwareInstallRequestFunc(ctx, hostID, softwareInstallerID, selfService, policyID) + return s.InsertSoftwareInstallRequestFunc(ctx, hostID, softwareInstallerID, opts) } func (s *DataStore) InsertSoftwareUninstallRequest(ctx context.Context, executionID string, hostID uint, softwareInstallerID uint) error { @@ -4605,7 +4610,7 @@ func (s *DataStore) MarkActivitiesAsStreamed(ctx context.Context, activityIDs [] return s.MarkActivitiesAsStreamedFunc(ctx, activityIDs) } -func (s *DataStore) ListHostUpcomingActivities(ctx context.Context, hostID uint, opt fleet.ListOptions) ([]*fleet.Activity, *fleet.PaginationMetadata, error) { +func (s *DataStore) ListHostUpcomingActivities(ctx context.Context, hostID uint, opt fleet.ListOptions) ([]*fleet.UpcomingActivity, *fleet.PaginationMetadata, error) { s.mu.Lock() s.ListHostUpcomingActivitiesFuncInvoked = true s.mu.Unlock() @@ -6558,13 +6563,6 @@ func (s *DataStore) NewHostScriptExecutionRequest(ctx context.Context, request * return s.NewHostScriptExecutionRequestFunc(ctx, request) } -func (s *DataStore) NewInternalScriptExecutionRequest(ctx context.Context, request *fleet.HostScriptRequestPayload) (*fleet.HostScriptResult, error) { - s.mu.Lock() - s.NewInternalScriptExecutionRequestFuncInvoked = true - s.mu.Unlock() - return s.NewInternalScriptExecutionRequestFunc(ctx, request) -} - func (s *DataStore) SetHostScriptExecutionResult(ctx context.Context, result *fleet.HostScriptResultPayload) (hsr *fleet.HostScriptResult, action string, err error) { s.mu.Lock() s.SetHostScriptExecutionResultFuncInvoked = true @@ -6586,6 +6584,13 @@ func (s *DataStore) ListPendingHostScriptExecutions(ctx context.Context, hostID return s.ListPendingHostScriptExecutionsFunc(ctx, hostID, onlyShowInternal) } +func (s *DataStore) ListReadyToExecuteScriptsForHost(ctx context.Context, hostID uint, onlyShowInternal bool) ([]*fleet.HostScriptResult, error) { + s.mu.Lock() + s.ListReadyToExecuteScriptsForHostFuncInvoked = true + s.mu.Unlock() + return s.ListReadyToExecuteScriptsForHostFunc(ctx, hostID, onlyShowInternal) +} + func (s *DataStore) NewScript(ctx context.Context, script *fleet.Script) (*fleet.Script, error) { s.mu.Lock() s.NewScriptFuncInvoked = true @@ -6754,6 +6759,13 @@ func (s *DataStore) ListPendingSoftwareInstalls(ctx context.Context, hostID uint return s.ListPendingSoftwareInstallsFunc(ctx, hostID) } +func (s *DataStore) ListReadyToExecuteSoftwareInstalls(ctx context.Context, hostID uint) ([]string, error) { + s.mu.Lock() + s.ListReadyToExecuteSoftwareInstallsFuncInvoked = true + s.mu.Unlock() + return s.ListReadyToExecuteSoftwareInstallsFunc(ctx, hostID) +} + func (s *DataStore) GetHostLastInstallData(ctx context.Context, hostID uint, installerID uint) (*fleet.HostLastInstallData, error) { s.mu.Lock() s.GetHostLastInstallDataFuncInvoked = true @@ -6957,11 +6969,11 @@ func (s *DataStore) InsertVPPAppWithTeam(ctx context.Context, app *fleet.VPPApp, return s.InsertVPPAppWithTeamFunc(ctx, app, teamID) } -func (s *DataStore) InsertHostVPPSoftwareInstall(ctx context.Context, hostID uint, appID fleet.VPPAppID, commandUUID string, associatedEventID string, selfService bool, policyID *uint) error { +func (s *DataStore) InsertHostVPPSoftwareInstall(ctx context.Context, hostID uint, appID fleet.VPPAppID, commandUUID string, associatedEventID string, opts fleet.HostSoftwareInstallOptions) error { s.mu.Lock() s.InsertHostVPPSoftwareInstallFuncInvoked = true s.mu.Unlock() - return s.InsertHostVPPSoftwareInstallFunc(ctx, hostID, appID, commandUUID, associatedEventID, selfService, policyID) + return s.InsertHostVPPSoftwareInstallFunc(ctx, hostID, appID, commandUUID, associatedEventID, opts) } func (s *DataStore) GetPastActivityDataForVPPAppInstall(ctx context.Context, commandResults *mdm.CommandResults) (*fleet.User, *fleet.ActivityInstalledAppStoreApp, error) { diff --git a/server/service/activities.go b/server/service/activities.go index d8d40f1d4fb8..977e93a2e9fb 100644 --- a/server/service/activities.go +++ b/server/service/activities.go @@ -150,7 +150,7 @@ type listHostUpcomingActivitiesRequest struct { type listHostUpcomingActivitiesResponse struct { Meta *fleet.PaginationMetadata `json:"meta"` - Activities []*fleet.Activity `json:"activities"` + Activities []*fleet.UpcomingActivity `json:"activities"` Count uint `json:"count"` Err error `json:"error,omitempty"` } @@ -169,7 +169,7 @@ func listHostUpcomingActivitiesEndpoint(ctx context.Context, request interface{} // ListHostUpcomingActivities returns a slice of upcoming activities for the // specified host. -func (svc *Service) ListHostUpcomingActivities(ctx context.Context, hostID uint, opt fleet.ListOptions) ([]*fleet.Activity, *fleet.PaginationMetadata, error) { +func (svc *Service) ListHostUpcomingActivities(ctx context.Context, hostID uint, opt fleet.ListOptions) ([]*fleet.UpcomingActivity, *fleet.PaginationMetadata, error) { // First ensure the user has access to list hosts, then check the specific // host once team_id is loaded. if err := svc.authz.Authorize(ctx, &fleet.Host{}, fleet.ActionList); err != nil { @@ -186,8 +186,9 @@ func (svc *Service) ListHostUpcomingActivities(ctx context.Context, hostID uint, // cursor-based pagination is not supported for upcoming activities opt.After = "" - // custom ordering is not supported, always by date (oldest first) - opt.OrderKey = "created_at" + // custom ordering is not supported, always by upcoming queue order + // (acual order is in the query, not set via ListOptions) + opt.OrderKey = "" opt.OrderDirection = fleet.OrderAscending // no matching query support opt.MatchQuery = "" diff --git a/server/service/hosts_test.go b/server/service/hosts_test.go index bb7f49439bf8..d115b4af217c 100644 --- a/server/service/hosts_test.go +++ b/server/service/hosts_test.go @@ -653,7 +653,7 @@ func TestHostAuth(t *testing.T) { ds.SetOrUpdateCustomHostDeviceMappingFunc = func(ctx context.Context, hostID uint, email, source string) ([]*fleet.HostDeviceMapping, error) { return nil, nil } - ds.ListHostUpcomingActivitiesFunc = func(ctx context.Context, hostID uint, opt fleet.ListOptions) ([]*fleet.Activity, *fleet.PaginationMetadata, error) { + ds.ListHostUpcomingActivitiesFunc = func(ctx context.Context, hostID uint, opt fleet.ListOptions) ([]*fleet.UpcomingActivity, *fleet.PaginationMetadata, error) { return nil, nil, nil } ds.GetHostLockWipeStatusFunc = func(ctx context.Context, host *fleet.Host) (*fleet.HostLockWipeStatus, error) { diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index c7fa9e838bd5..bcbc3dc4921f 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -12190,7 +12190,7 @@ func (s *integrationTestSuite) TestListHostUpcomingActivities() { require.NoError(t, err) s1Meta, err := s.ds.GetSoftwareInstallerMetadataByID(ctx, sw1) require.NoError(t, err) - h1Foo, err := s.ds.InsertSoftwareInstallRequest(ctx, host1.ID, s1Meta.InstallerID, false, nil) + h1Foo, err := s.ds.InsertSoftwareInstallRequest(ctx, host1.ID, s1Meta.InstallerID, fleet.HostSoftwareInstallOptions{}) require.NoError(t, err) // force an order to the activities diff --git a/server/service/integration_mdm_dep_test.go b/server/service/integration_mdm_dep_test.go index 939a3e00fea9..d5b2f8af00d9 100644 --- a/server/service/integration_mdm_dep_test.go +++ b/server/service/integration_mdm_dep_test.go @@ -2170,7 +2170,7 @@ func (s *integrationMDMTestSuite) TestSetupExperienceFlowWithSoftwareAndScriptAu require.NotNil(t, getSoftwareTitleResp.SoftwareTitle) require.NotNil(t, getSoftwareTitleResp.SoftwareTitle.SoftwarePackage) - debugPrintActivities := func(activities []*fleet.Activity) []string { + debugPrintActivities := func(activities []*fleet.UpcomingActivity) []string { var res []string for _, activity := range activities { res = append(res, fmt.Sprintf("%+v", activity)) diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index db9173777858..fb4ebdd0d931 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -11605,7 +11605,7 @@ func (s *integrationMDMTestSuite) TestVPPApps() { var hostActivitiesResp listHostUpcomingActivitiesResponse s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/activities/upcoming", installHost.ID), nil, http.StatusOK, &hostActivitiesResp) - activitiesToString := func(activities []*fleet.Activity) []string { + activitiesToString := func(activities []*fleet.UpcomingActivity) []string { var res []string for _, activity := range activities { res = append(res, fmt.Sprintf("%+v", activity)) @@ -12028,7 +12028,7 @@ func (s *integrationMDMTestSuite) TestVPPAppPolicyAutomation() { var hostActivitiesResp listHostUpcomingActivitiesResponse s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/activities/upcoming", mdmHost.ID), nil, http.StatusOK, &hostActivitiesResp) - activitiesToString := func(activities []*fleet.Activity) []string { + activitiesToString := func(activities []*fleet.UpcomingActivity) []string { var res []string for _, activity := range activities { res = append(res, fmt.Sprintf("%+v", activity)) diff --git a/server/service/orbit.go b/server/service/orbit.go index 21502285cba7..e082a41da808 100644 --- a/server/service/orbit.go +++ b/server/service/orbit.go @@ -294,8 +294,8 @@ func (svc *Service) GetOrbitConfig(ctx context.Context) (fleet.OrbitConfig, erro } } - // load the pending script executions for that host - pending, err := svc.ds.ListPendingHostScriptExecutions(ctx, host.ID, appConfig.ServerSettings.ScriptsDisabled) + // load the (active, ready to execute) pending script executions for that host + pending, err := svc.ds.ListReadyToExecuteScriptsForHost(ctx, host.ID, appConfig.ServerSettings.ScriptsDisabled) if err != nil { return fleet.OrbitConfig{}, err } @@ -310,7 +310,8 @@ func (svc *Service) GetOrbitConfig(ctx context.Context) (fleet.OrbitConfig, erro notifs.RunDiskEncryptionEscrow = host.IsLUKSSupported() && host.DiskEncryptionEnabled != nil && *host.DiskEncryptionEnabled && svc.ds.IsHostPendingEscrow(ctx, host.ID) - pendingInstalls, err := svc.ds.ListPendingSoftwareInstalls(ctx, host.ID) + // load the (active, ready to execute) pending software install executions for that host + pendingInstalls, err := svc.ds.ListReadyToExecuteSoftwareInstalls(ctx, host.ID) if err != nil { return fleet.OrbitConfig{}, err } diff --git a/server/service/osquery.go b/server/service/osquery.go index 07e63c999da9..dfcbcc74ea12 100644 --- a/server/service/osquery.go +++ b/server/service/osquery.go @@ -1835,8 +1835,10 @@ func (svc *Service) processSoftwareForNewlyFailingPolicies( installUUID, err := svc.ds.InsertSoftwareInstallRequest( ctx, hostID, installerMetadata.InstallerID, - false, // Set Self-service as false because this is triggered by Fleet. - &policyID, + fleet.HostSoftwareInstallOptions{ + SelfService: false, + PolicyID: &policyID, + }, ) if err != nil { return ctxerr.Wrapf(ctx, err, @@ -1973,7 +1975,10 @@ func (svc *Service) processVPPForNewlyFailingPolicies( continue } - commandUUID, err := svc.EnterpriseOverrides.InstallVPPAppPostValidation(ctx, host, vppMetadata, vppToken, false, &policyID) + commandUUID, err := svc.EnterpriseOverrides.InstallVPPAppPostValidation(ctx, host, vppMetadata, vppToken, fleet.HostSoftwareInstallOptions{ + SelfService: false, + PolicyID: &policyID, + }) if err != nil { level.Error(svc.logger).Log( "msg", "failed to get install VPP app", diff --git a/server/service/software_installers.go b/server/service/software_installers.go index 5d7a1ffe58d5..783fc9c04be3 100644 --- a/server/service/software_installers.go +++ b/server/service/software_installers.go @@ -563,7 +563,7 @@ func (svc *Service) GetVPPTokenIfCanInstallVPPApps(ctx context.Context, appleDev return "", fleet.ErrMissingLicense // called downstream of auth checks so doesn't need skipauth } -func (svc *Service) InstallVPPAppPostValidation(ctx context.Context, host *fleet.Host, vppApp *fleet.VPPApp, token string, selfService bool, policyID *uint) (string, error) { +func (svc *Service) InstallVPPAppPostValidation(ctx context.Context, host *fleet.Host, vppApp *fleet.VPPApp, token string, opts fleet.HostSoftwareInstallOptions) (string, error) { return "", fleet.ErrMissingLicense // called downstream of auth checks so doesn't need skipauth }